diff options
Diffstat (limited to 'configserver/src')
407 files changed, 28038 insertions, 0 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ActivateLock.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ActivateLock.java new file mode 100644 index 00000000000..bcc920614ec --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ActivateLock.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorLock; + +import java.util.concurrent.TimeUnit; + +/** + * A lock to protect session activation. + * + * @author lulf + * @since 5.1 + */ +public class ActivateLock { + private static final String ACTIVATE_LOCK_NAME = "activateLock"; + private final CuratorLock curatorLock; + + public ActivateLock(Curator curator, Path rootPath) { + this.curatorLock = new CuratorLock(curator, rootPath.append(ACTIVATE_LOCK_NAME).getAbsolute()); + } + + public synchronized void acquire(TimeoutBudget timeoutBudget, boolean ignoreLockError) { + try { + curatorLock.tryLock(timeoutBudget.timeLeft().toMillis(), TimeUnit.MILLISECONDS); + } catch (Exception e) { + if (!ignoreLockError) { + throw new RuntimeException(e); + } + } + } + + public synchronized void release() { + if (curatorLock.hasLock()) { + curatorLock.unlock(); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationMapper.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationMapper.java new file mode 100644 index 00000000000..6ff0fa49593 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationMapper.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.http.NotFoundException; + +/** + * Used during config request handling to route to the right config model + * based on application id and version. + * @author Vegard Sjonfjell + */ +public final class ApplicationMapper { + + private final Map<ApplicationId, ApplicationSet> requestHandlers = new ConcurrentHashMap<>(); + + private ApplicationSet getApplicationSet(ApplicationId applicationId) { + ApplicationSet list = requestHandlers.get(applicationId); + if (list != null) { + return list; + } + + throw new NotFoundException("No such application id: " + applicationId); + } + + /** + * Register a Application to an application id and specific vespa version + */ + public void register(ApplicationId applicationId, ApplicationSet applicationSet) { + requestHandlers.put(applicationId, applicationSet); + } + + /** + * Remove all applications associated with this application id + */ + public void remove(ApplicationId applicationId) { + requestHandlers.remove(applicationId); + } + + /** + * Retrieve the Application corresponding to this application id and specific vespa version. + * + * @return the matching application, or null if none matches + */ + public Application getForVersion(ApplicationId applicationId, Optional<Version> vespaVersion) throws VersionDoesNotExistException { + return getApplicationSet(applicationId).getForVersionOrLatest(vespaVersion); + } + + /** Returns whether this registry has an application for the given application id */ + public boolean hasApplication(ApplicationId applicationId) { + return hasApplicationForVersion(applicationId, Optional.<Version>empty()); + } + + /** Returns whether this registry has an application for the given application id and vespa version */ + public boolean hasApplicationForVersion(ApplicationId applicationId, Optional<Version> vespaVersion) { + try { + return getForVersion(applicationId, vespaVersion) != null; + } + catch (VersionDoesNotExistException | NotFoundException ex) { + return false; + } + } + + /** + * Get the number of applications registered + */ + public int numApplications() { + return requestHandlers.size(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationSet.java new file mode 100644 index 00000000000..dd41ef2d489 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationSet.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.application.Application; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Immutable set of {@link Application}s with the same {@link ApplicationId}. With methods for getting defaults. + * + * @author vegard + */ +public final class ApplicationSet { + + private final Version latestVersion; + // TODO: Should not need these as part of application? + private final ApplicationId applicationId; + private final long generation; + private final HashMap<Version, Application> applications = new HashMap<>(); + + private ApplicationSet(List<Application> applicationList) { + applicationId = applicationList.get(0).getId(); + generation = applicationList.get(0).getApplicationGeneration(); + for (Application application : applicationList) { + applications.put(application.getVespaVersion(), application); + if (!application.getId().equals(applicationId)) { + throw new IllegalArgumentException("Trying to create set with different application ids"); + } + } + latestVersion = applications.keySet().stream().max((a, b) -> a.compareTo(b)).get(); + } + + public Application getForVersionOrLatest(Optional<Version> optionalVersion) { + return resolveForVersion(optionalVersion.orElse(latestVersion)); + } + + private Application resolveForVersion(Version vespaVersion) { + Application application = applications.get(vespaVersion); + if (application != null) + return application; + + // Does the latest version specify we can use it regardless? + Application latest = applications.get(latestVersion); + if (latest.getModel().allowModelVersionMismatch()) + return latest; + + throw new VersionDoesNotExistException(String.format("No application with vespa version %s exists", vespaVersion.toString())); + } + + public ApplicationId getId() { + return applicationId; + } + + public static ApplicationSet fromList(List<Application> applications) { + return new ApplicationSet(applications); + } + + public static ApplicationSet fromSingle(Application application) { + return fromList(Arrays.asList(application)); + } + + public Collection<String> getAllHosts() { + return applications.values().stream() + .flatMap(app -> app.getModel().getHosts().stream() + .map(HostInfo::getHostname)) + .collect(Collectors.toList()); + } + + public void updateHostMetrics() { + for (Application application : applications.values()) { + application.updateHostMetrics(application.getModel().getHosts().size()); + } + } + + public long getApplicationGeneration() { + return generation; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/CompressedApplicationInputStream.java new file mode 100644 index 00000000000..b8da2448c5b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/CompressedApplicationInputStream.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.io.ByteStreams; +import com.google.common.io.Files; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.vespa.config.server.http.SessionCreate; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; + +import java.io.*; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; + +/** + * A compressed application points to an application package that can be decompressed. + * + * @author lulf + * @since 5.1 + */ +public class CompressedApplicationInputStream implements AutoCloseable { + private static final Logger log = Logger.getLogger(CompressedApplicationInputStream.class.getPackage().getName()); + private final ArchiveInputStream ais; + + /** + * Create an instance of a compressed application from an input stream. + * + * @param is the input stream containing the compressed files. + * @param contentType the content type for determining what kind of compressed stream should be used. + * @return An instance of an unpacked application. + */ + public static CompressedApplicationInputStream createFromCompressedStream(InputStream is, String contentType) { + try { + ArchiveInputStream ais = getArchiveInputStream(is, contentType); + return createFromCompressedStream(ais); + } catch (IOException e) { + throw new InternalServerException("Unable to create compressed application stream", e); + } + } + + public static CompressedApplicationInputStream createFromCompressedStream(ArchiveInputStream ais) { + return new CompressedApplicationInputStream(ais); + } + + private static ArchiveInputStream getArchiveInputStream(InputStream is, String contentTypeHeader) throws IOException { + ArchiveInputStream ais; + switch (contentTypeHeader) { + case SessionCreate.APPLICATION_X_GZIP: + ais = new TarArchiveInputStream(new GZIPInputStream(is)); + break; + case SessionCreate.APPLICATION_ZIP: + ais = new ZipArchiveInputStream(is); + break; + default: + throw new BadRequestException("Unable to decompress"); + } + return ais; + } + + private CompressedApplicationInputStream(ArchiveInputStream ais) { + this.ais = ais; + } + + /** + * Close this stream. + * @throws IOException if the stream could not be closed + */ + public void close() throws IOException { + ais.close(); + } + + File decompress() throws IOException { + return decompress(Files.createTempDir()); + } + + public File decompress(File dir) throws IOException { + decompressInto(dir); + dir = findActualApplicationDir(dir); + return dir; + } + + private void decompressInto(File application) throws IOException { + log.log(LogLevel.DEBUG, "Application is in " + application.getAbsolutePath()); + int entries = 0; + ArchiveEntry entry; + while ((entry = ais.getNextEntry()) != null) { + log.log(LogLevel.DEBUG, "Unpacking " + entry.getName()); + File outFile = new File(application, entry.getName()); + // FIXME/TODO: write more tests that break this logic. I have a feeling it is not very robust. + if (entry.isDirectory()) { + if (!(outFile.exists() && outFile.isDirectory())) { + log.log(LogLevel.DEBUG, "Creating dir: " + outFile.getAbsolutePath()); + boolean res = outFile.mkdirs(); + if (!res) { + log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); + } + } + } else { + log.log(LogLevel.DEBUG, "Creating output file: " + outFile.getAbsolutePath()); + + // Create parent dir if necessary + String parent = outFile.getParent(); + new File(parent).mkdirs(); + + FileOutputStream fos = new FileOutputStream(outFile); + ByteStreams.copy(ais, fos); + fos.close(); + } + entries++; + } + if (entries == 0) { + log.log(LogLevel.WARNING, "Not able to read any entries from " + application.getName()); + } + } + + private File findActualApplicationDir(File application) { + // If application is in e.g. application/, use that as root for UnpackedApplication + File[] files = application.listFiles(); + if (files != null && files.length == 1 && files[0].isDirectory()) { + application = files[0]; + } + return application; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigResponseFactory.java new file mode 100644 index 00000000000..3fc3c0ff8aa --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigResponseFactory.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.protocol.ConfigResponse; + +/** + * Represents a component that creates config responses from a payload. Different implementations + * can do transformations of the payload such as compression. + * + * @author lulf + * @since 5.19 + */ +public interface ConfigResponseFactory { + /** + * Create a {@link ConfigResponse} for a given payload and generation. + * @param payload The {@link com.yahoo.vespa.config.ConfigPayload} to put in the response. + * @param defFile The {@link com.yahoo.config.codegen.InnerCNode} def file for this config. + * @param generation The payload generation. @return A {@link ConfigResponse} that can be sent to the client. + */ + ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigResponseFactoryFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigResponseFactoryFactory.java new file mode 100644 index 00000000000..309f9052a71 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigResponseFactoryFactory.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; + +/** + * Logic to select the appropriate response factory based on config. + * TODO: Move this to {@link ConfigResponseFactory} when we have java 8. + * + * @author lulf + * @since 5.20 + */ +public class ConfigResponseFactoryFactory { + public static ConfigResponseFactory createFactory(ConfigserverConfig configserverConfig) { + switch (configserverConfig.payloadCompressionType()) { + case LZ4: + return new LZ4ConfigResponseFactory(); + case UNCOMPRESSED: + return new UncompressedConfigResponseFactory(); + default: + throw new IllegalArgumentException("Unknown payload compression type " + configserverConfig.payloadCompressionType()); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java new file mode 100644 index 00000000000..b4a0f6135c3 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.Deployer; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.version.VersionState; + +/** + * Main component that bootstraps and starts config server threads. + * + * @author lulf + * @since 5.1 + */ +public class ConfigServerBootstrap extends AbstractComponent implements Runnable { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerBootstrap.class.getName()); + private final RpcServer server; + private final Thread serverThread; + + // The tenants object is injected so that all initial requests handlers are + // added to the rpcserver before it starts answering rpc requests. + @SuppressWarnings("UnusedParameters") + @Inject + public ConfigServerBootstrap(Tenants tenants, RpcServer server, Deployer deployer, VersionState versionState) { + this.server = server; + if (versionState.isUpgraded()) { + log.log(LogLevel.INFO, "Configserver upgraded from " + versionState.storedVersion() + " to " + versionState.currentVersion() + ". Redeploying all applications"); + tenants.redeployApplications(deployer); + log.log(LogLevel.INFO, "All applications redeployed"); + } + versionState.saveNewVersion(); + this.serverThread = new Thread(this, "configserver main"); + serverThread.start(); + } + + @Override + public void deconstruct() { + log.log(LogLevel.INFO, "Stopping config server"); + server.stop(); + try { + serverThread.join(); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); + } + } + + @Override + public void run() { + log.log(LogLevel.DEBUG, "Starting RPC server"); + server.run(); + log.log(LogLevel.DEBUG, "RPC server stopped"); + } + +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java new file mode 100644 index 00000000000..b589e96e2e9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.application.provider.Bundle; +import com.yahoo.config.application.ConfigDefinitionDir; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Config server db is the maintainer of the serverdb directory containing def files and the file system sessions. + * See also {@link com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs} which maintains directories per tenant. + * + * @author lulf + * @since 5.1 + */ +public class ConfigServerDB { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigServerDB.class.getName()); + private final File serverDB; + private final ConfigserverConfig configserverConfig; + + public ConfigServerDB(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; + this.serverDB = new File(Defaults.getDefaults().underVespaHome(configserverConfig.configServerDBDir())); + create(); + try { + initialize(configserverConfig.configModelPluginDir()); + } catch (IllegalArgumentException e) { + log.log(LogLevel.ERROR, "Error initializing serverdb: " + e.getMessage()); + } catch (IOException e) { + throw new RuntimeException("Unable to initialize server db", e); + } + } + + public static ConfigServerDB createTestConfigServerDb(String dir) { + return new ConfigServerDB(new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(dir))); + } + + public File dest() { return new File(serverDB, "configs"); } + public File classes() { return new File(serverDB, "classes"); } + public File vespaapps() { return new File(serverDB, "vespaapps"); } + public File serverdefs() { return new File(serverDB, "serverdefs"); } + + + /** + * Creates all the config server db's dirs that are global. + */ + public void create() { + cr(dest()); + cr(classes()); + cr(vespaapps()); + cr(serverdefs()); + } + + public static void cr(File d) { + if (d.exists()) { + if (!d.isDirectory()) { + throw new IllegalArgumentException(d.getAbsolutePath() + " exists, but isn't a directory."); + } + } else { + if (!d.mkdirs()) { + throw new IllegalArgumentException("Couldn't create " + d.getAbsolutePath()); + } + } + } + + private void initialize(List<String> pluginDirectories) throws IOException { + IOUtils.recursiveDeleteDir(serverdefs()); + IOUtils.copyDirectory(classes(), serverdefs()); + ConfigDefinitionDir configDefinitionDir = new ConfigDefinitionDir(serverdefs()); + ArrayList<Bundle> bundles = new ArrayList<>(); + for (String pluginDirectory : pluginDirectories) { + bundles.addAll(Bundle.getBundles(new File(pluginDirectory))); + } + log.log(LogLevel.DEBUG, "Found " + bundles.size() + " bundles"); + List<Bundle> addedBundles = new ArrayList<>(); + for (Bundle bundle : bundles) { + log.log(LogLevel.DEBUG, "Bundle in " + bundle.getFile().getAbsolutePath() + " appears to contain " + bundle.getDefEntries().size() + " entries"); + configDefinitionDir.addConfigDefinitionsFromBundle(bundle, addedBundles); + addedBundles.add(bundle); + } + } + + public ConfigserverConfig getConfigserverConfig() { + return configserverConfig; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerSpec.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerSpec.java new file mode 100644 index 00000000000..a84985d738c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerSpec.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author tonytv + */ +public class ConfigServerSpec implements com.yahoo.config.model.api.ConfigServerSpec { + private final String hostName; + private final int configServerPort; + private final int httpPort; + private final int zooKeeperPort; + + public String getHostName() { + return hostName; + } + + public int getConfigServerPort() { + return configServerPort; + } + + public int getHttpPort() { + return httpPort; + } + + public int getZooKeeperPort() { + return zooKeeperPort; + } + + @Override + public boolean equals(Object o) { + if (o instanceof ConfigServerSpec) { + ConfigServerSpec other = (ConfigServerSpec)o; + + return hostName.equals(other.hostName) && + configServerPort == other.configServerPort && + httpPort == other.httpPort && + zooKeeperPort == other.zooKeeperPort; + } else { + return false; + } + } + + @Override + public int hashCode() { + return hostName.hashCode(); + } + + public ConfigServerSpec(String hostName, int configServerPort, int httpPort, int zooKeeperPort) { + this.hostName = hostName; + this.configServerPort = configServerPort; + this.httpPort = httpPort; + this.zooKeeperPort = zooKeeperPort; + } + + public static List<com.yahoo.config.model.api.ConfigServerSpec> fromConfig(ConfigserverConfig configserverConfig) { + List<com.yahoo.config.model.api.ConfigServerSpec> specs = new ArrayList<>(); + for (ConfigserverConfig.Zookeeperserver server : configserverConfig.zookeeperserver()) { + // TODO We cannot be sure that http port always is rpcport + 1 + specs.add(new ConfigServerSpec(server.hostname(), configserverConfig.rpcport(), configserverConfig.rpcport() + 1, server.port())); + } + return specs; + } + + @Override + public String toString() { + return "hostname=" + hostName + ", rpc port=" + configServerPort + ", http port=" + httpPort + ", zookeeper port=" + zooKeeperPort; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/DelayedConfigResponses.java new file mode 100644 index 00000000000..1c6f75b62f0 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/DelayedConfigResponses.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.TargetWatcher; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.config.provision.ApplicationId; + +import java.util.*; +import java.util.concurrent.*; +import java.util.logging.Logger; + +/** + * Takes care of <i>delayed responses</i> in the config server. + * A delayed response is a response sent at request (server) timeout + * for a config which has not changed since the request was initiated. + * + * @author musum + */ +public class DelayedConfigResponses { + private static final Logger log = Logger.getLogger(DelayedConfigResponses.class.getName()); + private final RpcServer rpcServer; + + private final ScheduledExecutorService executorService; + private final boolean useJrtWatcher; + + private Map<ApplicationId, MetricUpdater> metrics = new ConcurrentHashMap<>(); + + /* Requests that resolve to config that has not changed are put on this queue. When reloading + config, all requests on this queue are reprocessed as if they were a new request */ + private final Map<ApplicationId, BlockingQueue<DelayedConfigResponse>> delayedResponses = + new ConcurrentHashMap<>(); + + public DelayedConfigResponses(RpcServer rpcServer, int numTimerThreads) { + this(rpcServer, numTimerThreads, true); + } + + // Since JRT does not allow adding watcher for "fake" requests, we must be able to disable it for unit tests :( + DelayedConfigResponses(RpcServer rpcServer, int numTimerThreads, boolean useJrtWatcher) { + this.rpcServer = rpcServer; + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(numTimerThreads, ThreadFactoryFactory.getThreadFactory(DelayedConfigResponses.class.getName())); + executor.setRemoveOnCancelPolicy(true); + this.executorService = executor; + this.useJrtWatcher = useJrtWatcher; + } + + List<DelayedConfigResponse> allDelayedResponses() { + List<DelayedConfigResponse> responses = new ArrayList<>(); + for (Map.Entry<ApplicationId, BlockingQueue<DelayedConfigResponse>> entry : delayedResponses.entrySet()) { + responses.addAll(entry.getValue()); + } + return responses; + } + + /** + * The run method of this class is run by a Timer when the timeout expires. + * The timer associated with this response must be cancelled first. + */ + public class DelayedConfigResponse implements Runnable, TargetWatcher { + + final JRTServerConfigRequest request; + private final BlockingQueue<DelayedConfigResponse> delayedResponsesQueue; + private final ApplicationId app; + private ScheduledFuture<?> future; + + public DelayedConfigResponse(JRTServerConfigRequest req, BlockingQueue<DelayedConfigResponse> delayedResponsesQueue, ApplicationId app) { + this.request = req; + this.delayedResponsesQueue = delayedResponsesQueue; + this.app = app; + } + + public synchronized void run() { + remove(); + rpcServer.addToRequestQueue(request, true, null); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, logPre()+"DelayedConfigResponse. putting on queue: " + request.getShortDescription()); + } + } + + /** + * Remove delayed response from its queue + */ + private void remove() { + delayedResponsesQueue.remove(this); + removeWatcher(); + } + + public JRTServerConfigRequest getRequest() { + return request; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Delayed response for ").append(logPre()).append(request.getShortDescription()); + return sb.toString(); + } + + public ApplicationId getAppId() { return app; } + + String logPre() { + return Tenants.logPre(app); + } + + public synchronized boolean cancel() { + remove(); + if (future == null) { + throw new IllegalStateException("Cannot cancel a task that has not been scheduled"); + } + return future.cancel(false); + } + + public synchronized void schedule(long delay) throws InterruptedException { + delayedResponsesQueue.put(this); + future = executorService.schedule(this, delay, TimeUnit.MILLISECONDS); + addWatcher(); + } + + /** + * Removes this delayed response if target is invalid. + * + * @param target a Target that has become invalid (i.e, client has closed connection) + * @see DelayedConfigResponses + */ + @Override + public void notifyTargetInvalid(Target target) { + cancel(); + } + + private void addWatcher() { + if (useJrtWatcher) { + request.getRequest().target().addWatcher(this); + } + } + + private void removeWatcher() { + if (useJrtWatcher) { + request.getRequest().target().removeWatcher(this); + } + } + } + + /** + * Creates a DelayedConfigResponse object for taking care of requests that should + * not be responded to right away. Puts the object on the delayedResponsesQueue. + * + * NOTE: This method is called from multiple threads, so everything here needs to be + * thread safe! + * + * @param request a JRTConfigRequest + */ + public final void delayResponse(JRTServerConfigRequest request, GetConfigContext context) { + if (request.isDelayedResponse()) { + log.log(LogLevel.DEBUG, context.logPre()+"Request already delayed"); + } else { + createQueueIfNotExists(context); + BlockingQueue<DelayedConfigResponse> delayedResponsesQueue = delayedResponses.get(context.applicationId()); + DelayedConfigResponse response = new DelayedConfigResponse(request, delayedResponsesQueue, context.applicationId()); + request.setDelayedResponse(true); + try { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, context.logPre()+"Putting on delayedRequests queue (" + delayedResponsesQueue.size() + " elements): " + + response.getRequest().getShortDescription()); + } + // Config will be resolved in the run() method of DelayedConfigResponse, + // when the timer expires or config is updated/reloaded. + response.schedule(Math.max(0, request.getTimeout())); + metricDelayedResponses(context.applicationId(), delayedResponsesQueue.size()); + } catch (InterruptedException e) { + log.log(LogLevel.WARNING, context.logPre()+"Interrupted when putting on delayed requests queue."); + } + } + } + + private synchronized void metricDelayedResponses(ApplicationId app, int elems) { + if ( ! metrics.containsKey(app)) { + metrics.put(app, rpcServer.metricUpdaterFactory().getOrCreateMetricUpdater(Metrics.createDimensions(app))); + } + metrics.get(app).setDelayedResponses(elems); + } + + private synchronized void createQueueIfNotExists(GetConfigContext context) { + if ( ! delayedResponses.containsKey(context.applicationId())) { + delayedResponses.put(context.applicationId(), new LinkedBlockingQueue<>()); + } + } + + public void stop() { + executorService.shutdown(); + } + + /** + * Drains delayed responses queue and returns responses in an array + * + * @return and array of DelayedConfigResponse objects + */ + public List<DelayedConfigResponse> drainQueue(ApplicationId app) { + ArrayList<DelayedConfigResponse> ret = new ArrayList<>(); + + if (delayedResponses.containsKey(app)) { + BlockingQueue<DelayedConfigResponse> queue = delayedResponses.get(app); + queue.drainTo(ret); + } + metrics.remove(app); + return ret; + } + + public String toString() { + return "DelayedConfigResponses. Average Size=" + size(); + } + + public int size() { + int totalQueueSize = 0; + int numQueues = 0; + for (Map.Entry<ApplicationId, BlockingQueue<DelayedConfigResponse>> e : delayedResponses.entrySet()) { + numQueues++; + totalQueueSize+=e.getValue().size(); + } + return (numQueues > 0) ? (totalQueueSize / numQueues) : 0; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/DeployHandlerLogger.java new file mode 100644 index 00000000000..2c652d74b97 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/DeployHandlerLogger.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; + +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A {@link DeployLogger} which persists messages as a {@link Slime} tree, and holds a tenant and application name. + * + * @author lulf + * @since 5.1 + */ +public class DeployHandlerLogger implements DeployLogger { + private static final Logger log = Logger.getLogger(DeployHandlerLogger.class.getName()); + + private final Cursor logroot; + private final boolean verbose; + private final ApplicationId app; + + public DeployHandlerLogger(Cursor root, boolean verbose, ApplicationId app) { + logroot = root; + this.verbose = verbose; + this.app = app; + } + + @Override + public void log(Level level, String message) { + if ((level == LogLevel.FINE || + level == LogLevel.DEBUG || + level == LogLevel.SPAM) && + !verbose) { + return; + } + String fullMsg = Tenants.logPre(app)+message; + Cursor entry = logroot.addObject(); + entry.setLong("time", System.currentTimeMillis()); + entry.setString("level", level.getName()); + entry.setString("message", fullMsg); + // Also tee to a normal log, Vespa log for example, but use level fine + log.log(LogLevel.FINE, fullMsg); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java new file mode 100644 index 00000000000..e27a1c56dc3 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; + +/** + * Contains the context for serving getconfig requests so that this information does not have to be looked up multiple times. + * + * @author lulf + * @since 5.8 + */ +public class GetConfigContext { + + private final ApplicationId app; + private final RequestHandler requestHandler; + private final Trace trace; + + private GetConfigContext(ApplicationId app, RequestHandler handler, Trace trace) { + this.app = app; + this.requestHandler = handler; + this.trace = trace; + } + + public TenantName tenant() { + return app.tenant(); + } + + public ApplicationId applicationId() { + return app; + } + + public Trace trace() { + return trace; + } + + public RequestHandler requestHandler() { + return requestHandler; + } + + public static GetConfigContext create(ApplicationId app, RequestHandler handler, Trace trace) { + return new GetConfigContext(app, handler, trace); + } + + public static GetConfigContext testContext(ApplicationId app) { + return new GetConfigContext(app, null, null); + } + + /** + * Helper to produce a log preamble with the tenant and app id + * @return log msg preamble + */ + public String logPre() { + return Tenants.logPre(app); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java new file mode 100644 index 00000000000..eb5fedf9a40 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigProcessor.java @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.jrt.Request; +import com.yahoo.log.LogLevel; +import com.yahoo.net.LinuxInetAddress; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.UnknownConfigIdException; +import com.yahoo.vespa.config.protocol.*; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.net.UnknownHostException; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** +* @author musum +* @since 5.1 +*/ +class GetConfigProcessor implements Runnable { + + private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName()); + private static final String localHostName; + + private final JRTServerConfigRequest request; + /* True only when this request has expired its server timeout and we need to respond to the client */ + private boolean forceResponse = false; + private final RpcServer rpcServer; + private String logPre = ""; + + GetConfigProcessor(RpcServer rpcServer, JRTServerConfigRequest request, boolean forceResponse) { + this.rpcServer = rpcServer; + this.request = request; + this.forceResponse = forceResponse; + } + + private void respond(JRTServerConfigRequest request) { + final Request req = request.getRequest(); + if (req.isError()) { + Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? LogLevel.DEBUG : LogLevel.INFO; + log.log(logLevel, logPre + req.errorMessage()); + } + rpcServer.respond(request); + } + + private void handleError(JRTServerConfigRequest request, int errorCode, String message) { + String target = "(unknown)"; + try { + target = request.getRequest().target().toString(); + } catch (IllegalStateException e) { + //ignore when no target + } + request.addErrorResponse(errorCode, logPre + "Failed request (" + message + ") from " + target); + respond(request); + } + + // TODO: Increment statistics (Metrics) failed counters when requests fail + public void run() { + //Request has already been detached + if (!request.validateParameters()) { + // Error code is set in verifyParameters if parameters are not OK. + log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); + respond(request); + return; + } + Trace trace = request.getRequestTrace(); + if (logDebug(trace)) { + debugLog(trace, "GetConfigProcessor.run() on " + localHostName); + } + + Optional<TenantName> tenant = rpcServer.resolveTenant(request, trace); + + // If we are certain that this request is from a node that no longer belongs to this application, + // fabricate an empty request to cause the sentinel to stop all running services + if (rpcServer.isHostedVespa() && rpcServer.allTenantsLoaded() && !tenant.isPresent() && isSentinelConfigRequest(request)) { + returnEmpty(request); + return; + } + + GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); + if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.<Version>empty())) { + handleError(request, ErrorCode.APPLICATION_NOT_LOADED, "No application exists"); + return; + } + + Optional<Version> vespaVersion = rpcServer.useRequestVersion() ? + request.getVespaVersion().map(VespaVersion::toString).map(Version::fromString) : + Optional.empty(); + if (logDebug(trace)) { + debugLog(trace, "Using version " + getPrintableVespaVersion(vespaVersion)); + } + + if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) { + handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + getPrintableVespaVersion(vespaVersion)); + return; + } + + this.logPre = Tenants.logPre(context.applicationId()); + ConfigResponse config; + try { + config = rpcServer.resolveConfig(request, context, vespaVersion); + } catch (UnknownConfigDefinitionException e) { + handleError(request, ErrorCode.UNKNOWN_DEFINITION, "Unknown config definition " + request.getConfigKey()); + return; + } catch (UnknownConfigIdException e) { + handleError(request, ErrorCode.ILLEGAL_CONFIGID, "Illegal config id " + request.getConfigKey().getConfigId()); + return; + } catch (Exception e) { + log.log(Level.SEVERE, "Unexpected error handling config request", e); + handleError(request, ErrorCode.INTERNAL_ERROR, "Internal error " + e.getMessage()); + return; + } + + // config == null is not an error, but indicates that the config will be returned later. + if ((config != null) && (!config.hasEqualConfig(request) || config.hasNewerGeneration(request) || forceResponse)) { + // debugLog(trace, "config response before encoding:" + config.toString()); + request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.getConfigMd5()); + if (logDebug(trace)) { + debugLog(trace, "return response: " + request.getShortDescription()); + } + respond(request); + } else { + if (logDebug(trace)) { + debugLog(trace, "delaying response " + request.getShortDescription()); + } + rpcServer.delayResponse(request, context); + } + } + + private boolean isSentinelConfigRequest(JRTServerConfigRequest request) { + return request.getConfigKey().getName().equals(SentinelConfig.getDefName()) && + request.getConfigKey().getNamespace().equals(SentinelConfig.getDefNamespace()); + } + + private static String getPrintableVespaVersion(Optional<Version> vespaVersion) { + return (vespaVersion.isPresent() ? vespaVersion.get().toString() : "LATEST"); + } + + private void returnEmpty(JRTServerConfigRequest request) { + ConfigPayload emptyPayload = ConfigPayload.empty(); + String configMd5 = ConfigUtils.getMd5(emptyPayload); + ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, configMd5); + request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), config.getConfigMd5()); + respond(request); + } + + /** + * Done in a static block to prevent people invoking this directly. + * Do not call java.net.Inet4AddressImpl.getLocalHostName() on each request, as this causes CPU bottlenecks. + */ + static { + String hostName = "unknown"; + try { + hostName = LinuxInetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + // ignore if it fails + } + localHostName = hostName; + } + + static boolean logDebug(Trace trace) { + return trace.shouldTrace(RpcServer.TRACELEVEL_DEBUG) || log.isLoggable(LogLevel.DEBUG); + } + + private void debugLog(Trace trace, String message) { + if (logDebug(trace)) { + log.log(LogLevel.DEBUG, logPre + message); + trace.trace(RpcServer.TRACELEVEL_DEBUG, logPre + message); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java new file mode 100644 index 00000000000..7ea65173a53 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.session.SessionPreparer; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; + +import java.util.Optional; + +/** + * Interface representing all global config server components used within the config server. + * + * @author lulf + * @since 5.1 + */ +public interface GlobalComponentRegistry { + + Curator getCurator(); + ConfigCurator getConfigCurator(); + Metrics getMetrics(); + ConfigServerDB getServerDB(); + SessionPreparer getSessionPreparer(); + ConfigserverConfig getConfigserverConfig(); + TenantListener getTenantListener(); + ReloadListener getReloadListener(); + SuperModelGenerationCounter getSuperModelGenerationCounter(); + ConfigDefinitionRepo getConfigDefinitionRepo(); + PermanentApplicationPackage getPermanentApplicationPackage(); + HostRegistries getHostRegistries(); + ModelFactoryRegistry getModelFactoryRegistry(); + Optional<Provisioner> getHostProvisioner(); + Zone getZone(); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/HostRegistries.java b/configserver/src/main/java/com/yahoo/vespa/config/server/HostRegistries.java new file mode 100644 index 00000000000..dc411626b39 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/HostRegistries.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; + +import java.util.HashMap; + +/** + * Component to hold host registries. + * + * @author musum + */ +public class HostRegistries { + + private final HostRegistry<TenantName> tenantHostRegistry = new HostRegistry<>(); + private final HashMap<TenantName, HostRegistry<ApplicationId>> applicationHostRegistries = new HashMap<>(); + + public HostRegistry<TenantName> getTenantHostRegistry() { + return tenantHostRegistry; + } + + public HostRegistry<ApplicationId> getApplicationHostRegistry(TenantName tenant) { + return applicationHostRegistries.get(tenant); + } + + public HostRegistry<ApplicationId> createApplicationHostRegistry(TenantName tenant) { + HostRegistry<ApplicationId> applicationIdHostRegistry = new HostRegistry<>(); + applicationHostRegistries.put(tenant, applicationIdHostRegistry); + return applicationIdHostRegistry; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/HostRegistry.java new file mode 100644 index 00000000000..a62e0059c2a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/HostRegistry.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; + +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; +import com.yahoo.log.LogLevel; + +/** + * A host registry that create mappings between some type T and a list of hosts, represented as + * strings. + * TODO: Maybe we should have a Host type, but using String for now. + * TODO: Is there a generalized version of this pattern? Need some sort mix of Bimap and Multimap + * + * @author lulf + * @since 5.3 + */ +public class HostRegistry<T> implements HostValidator<T> { + + private static final Logger log = Logger.getLogger(HostRegistry.class.getName()); + + private final Map<T, Collection<String>> key2HostsMap = new ConcurrentHashMap<>(); + private final Map<String, T> host2KeyMap = new ConcurrentHashMap<>(); + + public T getKeyForHost(String hostName) { + return host2KeyMap.get(hostName); + } + + public void update(T key, Collection<String> newHosts) { + verifyHosts(key, newHosts); + log.log(LogLevel.DEBUG, "Setting hosts for key(" + key + "), newHosts(" + newHosts + "), currentHosts(" + getCurrentHosts(key) + ")"); + Collection<String> removedHosts = getRemovedHosts(newHosts, getCurrentHosts(key)); + removeHosts(removedHosts); + addHosts(key, newHosts); + } + + public void verifyHosts(T key, Collection<String> newHosts) { + for (String host : newHosts) { + if (hostAlreadyTaken(host, key)) { + throw new IllegalArgumentException("'" + key + "' tried to allocate host '" + host + "', but the host is already taken by '" + host2KeyMap.get(host) + "'"); + } + } + } + + public void removeHostsForKey(T key) { + for (Iterator<Map.Entry<T, Collection<String>>> it = key2HostsMap.entrySet().iterator(); it.hasNext(); ) { + Map.Entry<T, Collection<String>> entry = it.next(); + if (entry.getKey().equals(key)) { + Collection<String> hosts = entry.getValue(); + it.remove(); + removeHosts(hosts); + } + } + } + + public Collection<String> getAllHosts() { + return Collections.unmodifiableCollection(new ArrayList<>(host2KeyMap.keySet())); + } + + Collection<String> getCurrentHosts(T key) { + return key2HostsMap.containsKey(key) ? new ArrayList<>(key2HostsMap.get(key)) : new ArrayList<String>(); + } + + private boolean hostAlreadyTaken(String host, T key) { + return host2KeyMap.containsKey(host) && !key.equals(host2KeyMap.get(host)); + } + + private static Collection<String> getRemovedHosts(final Collection<String> newHosts, Collection<String> previousHosts) { + return Collections2.filter(previousHosts, new Predicate<String>() { + @Override + public boolean apply(String host) { + return !newHosts.contains(host); + } + }); + } + + private void removeHosts(Collection<String> removedHosts) { + for (String host : removedHosts) { + log.log(LogLevel.DEBUG, "Removing " + host); + host2KeyMap.remove(host); + } + } + + private void addHosts(T key, Collection<String> newHosts) { + for (String host : newHosts) { + log.log(LogLevel.DEBUG, "Adding " + host); + host2KeyMap.put(host, key); + } + key2HostsMap.put(key, new ArrayList<>(newHosts)); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/HostValidator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/HostValidator.java new file mode 100644 index 00000000000..67292e281bb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/HostValidator.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.Collection; + +/** + * A read only host registry that has mappings from a host to some type T. + * strings. + * + * @author lulf + * @since 5.9 + */ +public interface HostValidator<T> { + + void verifyHosts(T key, Collection<String> newHosts); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java new file mode 100644 index 00000000000..fd5529cdffd --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.SessionPreparer; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; + +import java.util.Optional; + +/** + * Registry containing all the "static"/"global" components in a config server in one place. + * + * @author lulf + * @since 5.1 + */ +public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry { + + private final Curator curator; + private final ConfigCurator configCurator; + private final Metrics metrics; + private final ModelFactoryRegistry modelFactoryRegistry; + private final ConfigServerDB serverDB; + private final SessionPreparer sessionPreparer; + private final RpcServer rpcServer; + private final ConfigserverConfig configserverConfig; + private final SuperModelGenerationCounter superModelGenerationCounter; + private final ConfigDefinitionRepo defRepo; + private final PermanentApplicationPackage permanentApplicationPackage; + private final HostRegistries hostRegistries; + private final Optional<Provisioner> hostProvisioner; + private final Zone zone; + + @Inject + public InjectedGlobalComponentRegistry(Curator curator, + ConfigCurator configCurator, + Metrics metrics, + ModelFactoryRegistry modelFactoryRegistry, + ConfigServerDB serverDB, + SessionPreparer sessionPreparer, + RpcServer rpcServer, + ConfigserverConfig configserverConfig, + SuperModelGenerationCounter superModelGenerationCounter, + ConfigDefinitionRepo defRepo, + PermanentApplicationPackage permanentApplicationPackage, + HostRegistries hostRegistries, + HostProvisionerProvider hostProvisionerProvider, + Zone zone) { + this.curator = curator; + this.configCurator = configCurator; + this.metrics = metrics; + this.modelFactoryRegistry = modelFactoryRegistry; + this.serverDB = serverDB; + this.sessionPreparer = sessionPreparer; + this.rpcServer = rpcServer; + this.configserverConfig = configserverConfig; + this.superModelGenerationCounter = superModelGenerationCounter; + this.defRepo = defRepo; + this.permanentApplicationPackage = permanentApplicationPackage; + this.hostRegistries = hostRegistries; + this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); + this.zone = zone; + } + + @Override + public Curator getCurator() { return curator; } + @Override + public ConfigCurator getConfigCurator() { return configCurator; } + @Override + public Metrics getMetrics() { return metrics; } + @Override + public ConfigServerDB getServerDB() { return serverDB; } + @Override + public SessionPreparer getSessionPreparer() { return sessionPreparer; } + @Override + public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } + @Override + public TenantListener getTenantListener() { return rpcServer; } + @Override + public ReloadListener getReloadListener() { return rpcServer; } + @Override + public SuperModelGenerationCounter getSuperModelGenerationCounter() { return superModelGenerationCounter; } + @Override + public ConfigDefinitionRepo getConfigDefinitionRepo() { return defRepo; } + @Override + public PermanentApplicationPackage getPermanentApplicationPackage() { return permanentApplicationPackage; } + @Override + public HostRegistries getHostRegistries() { return hostRegistries; } + @Override + public ModelFactoryRegistry getModelFactoryRegistry() { return modelFactoryRegistry; } + + @Override + public Optional<Provisioner> getHostProvisioner() { + return hostProvisioner; + } + + @Override + public Zone getZone() { + return zone; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/LZ4ConfigResponseFactory.java new file mode 100644 index 00000000000..66a2dc32eea --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/LZ4ConfigResponseFactory.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.LZ4PayloadCompressor; +import com.yahoo.vespa.config.protocol.CompressionInfo; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.SlimeConfigResponse; +import com.yahoo.vespa.config.util.ConfigUtils; + +/** + * Compressor that compresses config payloads to lz4. + * + * @author lulf + * @since 5.19 + */ +public class LZ4ConfigResponseFactory implements ConfigResponseFactory { + + private static LZ4PayloadCompressor compressor = new LZ4PayloadCompressor(); + + @Override + public ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation) { + Utf8Array rawPayload = payload.toUtf8Array(true); + String configMd5 = ConfigUtils.getMd5(rawPayload); + CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength()); + Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes())); + return new SlimeConfigResponse(compressed, defFile, generation, configMd5, info); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/PathProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/PathProvider.java new file mode 100644 index 00000000000..ca715d66a05 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/PathProvider.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.path.Path; + +/** + * Temporary provider of root path for components that will soon get them injected from a parent class. + * + * @author lulf + * * @since 5.1.24 + */ +public class PathProvider { + private final Path root; + // Path for Vespa-related data stored in Zookeeper (subpaths are relative to this path) + // NOTE: This should not be exposed, as this path can be different in testing, depending on how we configure it. + private static final String APPS_ZK_NODE = "sessions"; + private static final String VESPA_ZK_PATH = "/vespa/config"; + //private static final String VESPA_ZK_PATH = "/config/v2/tenants/default"; + private static final String LIVEAPP_ZK_NODE = "liveapp"; + + @Inject + public PathProvider() { + root = Path.fromString(VESPA_ZK_PATH); + } + + public PathProvider(Path root) { + this.root = root; + } + + public Path getRoot() { + return root; + } + + public Path getSessionDirs() { + return root.append(APPS_ZK_NODE); + } + + public Path getSessionDir(long sessionId) { + return getSessionDirs().append(String.valueOf(sessionId)); + } + + public Path getLiveApp() { + return root.append(LIVEAPP_ZK_NODE); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java new file mode 100644 index 00000000000..8d7eb8a9d7c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; + +/** + * Interface representing a reload handler. + * + * @author lulf + * @since 5.1.24 + */ +public interface ReloadHandler { + /** + * Reload config with the one contained in the application. + * + * @param applicationSet The set of applications to set as active. + */ + public void reloadConfig(ApplicationSet applicationSet); + + /** + * Remove an application and resources related to it. + * + * @param applicationId to be removed + */ + public void removeApplication(ApplicationId applicationId); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java new file mode 100644 index 00000000000..f519a656c8f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadListener.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; + +import java.util.Collection; + +/** + * A ReloadListener is used to signal to a component that config has been + * reloaded. It only exists because the RpcServer cannot distinguish between a + * successful reload of a new application and a reload of the same application. + * + * @author lulf + * @since 5.1 + */ +public interface ReloadListener { + + /** + * Signal the listener that config has been reloaded. + * + * @param tenant Name of tenant for which config was reloaded. + * @param application the {@link com.yahoo.vespa.config.server.application.Application} that will be reloaded + */ + public void configReloaded(TenantName tenant, ApplicationSet application); + + /** + * Signal the listener that hosts used by by a particular tenant. + * + * @param tenant Name of tenant. + * @param newHosts a {@link Collection} of hosts used by tenant. + */ + void hostsUpdated(TenantName tenant, Collection<String> newHosts); + + /** + * Verify that given hosts are available for use by tenant. + * TODO: Does not belong here... + * + * @param tenant tenant that wants to allocate hosts. + * @param newHosts a {@link java.util.Collection} of hosts that tenant wants to allocate. + * @throws java.lang.IllegalArgumentException if one or more of the hosts are in use by another tenant. + */ + void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts); + + /** + * Notifies listener that application with id {@link ApplicationId} has been removed. + * + * @param applicationId The {@link ApplicationId} of the removed application. + */ + void applicationRemoved(ApplicationId applicationId); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java new file mode 100644 index 00000000000..691035839bb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.Optional; +import java.util.Set; + +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.protocol.ConfigResponse; + +/** + * Instances of this can serve misc config related requests + * + * @author lulf + * @since 5.1 + */ +public interface RequestHandler { + + /** + * Resolves a config. Mandatory subclass hook for getConfig(). + * @param appId The application id to use + * @param req a config request + * @param vespaVersion vespa version + * @return The resolved config if it exists, else null. + */ + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion); + + /** + * Lists all configs (name, configKey) in the config model. + * @param appId application id to use + * @param vespaVersion optional vespa version + * @param recursive If true descend into all levels + * @return set of keys + */ + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive); + + /** + * Lists all configs (name, configKey) of the given key. The config id of the key is interpreted as a prefix to match. + * @param appId application id to use + * @param vespaVersion optional vespa version + * @param key def key to match + * @param recursive If true descend into all levels + * @return set of keys + */ + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> key, boolean recursive); + + /** + * Lists all available configs produced + * @param appId application id to use + * @param vespaVersion optional vespa version + * @return set of keys + */ + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion); + + /** + * List all config ids present + * @param appId application id to use + * @param vespaVersion optional vespa version + * @return a Set containing all config ids available + */ + public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion); + + /** + * True if application loaded + * @param appId The application id to use + * @param vespaVersion optional vespa version + * @return true if app loaded + */ + boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion); + + /** + * Resolve {@link ApplicationId} for a given hostname. Returns a default {@link ApplicationId} if no applications + * are found for that host. + * + * @param hostName hostname of client. + * @return an {@link ApplicationId} instance. + */ + ApplicationId resolveApplicationId(String hostName); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/RotationsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/RotationsCache.java new file mode 100644 index 00000000000..2a686e2dee3 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/RotationsCache.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.path.Path; + +import com.yahoo.vespa.curator.Curator; + +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Rotations for an application. Persisted in ZooKeeper. + * + * @author musum + */ +public class RotationsCache { + + private final Path path; + private final Curator curator; + + public RotationsCache(Curator curator, Path tenantPath) { + this.curator = curator; + this.path = tenantPath.append("rotationsCache/"); + } + + public Set<Rotation> readRotationsFromZooKeeper(ApplicationId applicationId) { + ObjectMapper objectMapper = new ObjectMapper(); + Path fullPath = path.append(applicationId.serializedForm()); + Set<Rotation> ret = new LinkedHashSet<>(); + try { + if (curator != null && curator.exists(fullPath)) { + byte[] data = curator.framework().getData().forPath(fullPath.getAbsolute()); + if (data.length > 0) { + Set<String> rotationIds = objectMapper.readValue(data, new TypeReference<Set<String>>() { + }); + ret.addAll(rotationIds.stream().map(Rotation::new).collect(Collectors.toSet())); + } + } + } catch (Exception e) { + throw new RuntimeException("Error reading rotations from ZooKeeper (" + fullPath + ")", e); + } + return ret; + } + + public void writeRotationsToZooKeeper(ApplicationId applicationId, Set<Rotation> rotations) { + if (rotations.size() > 0) { + final ObjectMapper objectMapper = new ObjectMapper(); + final Path cachePath = path.append(applicationId.serializedForm()); + final String absolutePath = cachePath.getAbsolute(); + try { + curator.create(cachePath); + final Set<String> rotationIds = rotations.stream().map(Rotation::getId).collect(Collectors.toSet()); + final byte[] data = objectMapper.writeValueAsBytes(rotationIds); + curator.framework().setData().forPath(absolutePath, data); + } catch (Exception e) { + throw new RuntimeException("Error writing rotations to ZooKeeper (" + absolutePath + ")", e); + } + } + } + + public void deleteRotationFromZooKeeper(ApplicationId applicationId) { + curator.delete(path.append(applicationId.serializedForm())); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java new file mode 100644 index 00000000000..f49268dd800 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/RpcServer.java @@ -0,0 +1,407 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.jrt.Acceptor; +import com.yahoo.jrt.Int32Value; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.Method; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.JRTMethods; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.JRTConfigRequest; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * An RPC server class that handles the config protocol RPC method "getConfigV3". + * Mandatory hooks need to be implemented by subclasses. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class RpcServer implements Runnable, ReloadListener, TenantListener { + + static final int TRACELEVEL = 6; + static final int TRACELEVEL_DEBUG = 9; + private static final String THREADPOOL_NAME = "rpcserver worker pool"; + private static final long SHUTDOWN_TIMEOUT = 60; + private final Supervisor supervisor = new Supervisor(new Transport()); + private Spec spec = null; + private boolean running = false; + private final boolean useRequestVersion; + private final boolean hostedVespa; + + private static final Logger log = Logger.getLogger(RpcServer.class.getName()); + + final DelayedConfigResponses delayedConfigResponses; + + private final HostRegistry<TenantName> hostRegistry; + private final Map<TenantName, TenantHandlerProvider> tenantProviders = new ConcurrentHashMap<>(); + private final SuperModelController superModelController; + private final MetricUpdater metrics; + private final MetricUpdaterFactory metricUpdaterFactory; + + private final ThreadPoolExecutor executorService; + private volatile boolean allTenantsLoaded = false; + + /** + * Creates an RpcServer listening on the specified <code>port</code>. + * + * @param config The config to use for setting up this server + */ + @Inject + public RpcServer(ConfigserverConfig config, SuperModelController superModelController, MetricUpdaterFactory metrics, HostRegistries hostRegistries) { + this.superModelController = superModelController; + this.metricUpdaterFactory = metrics; + this.supervisor.setMaxOutputBufferSize(config.maxoutputbuffersize()); + this.metrics = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); + BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); + executorService = new ThreadPoolExecutor(config.numthreads(), config.numthreads(), 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); + delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); + spec = new Spec(null, config.rpcport()); + hostRegistry = hostRegistries.getTenantHostRegistry(); + this.useRequestVersion = config.useVespaVersionInRequest(); + this.hostedVespa = config.hostedVespa(); + setUpHandlers(); + } + + /** + * Handles RPC method "config.v3.getConfig" requests. + * Uses the template pattern to call methods in classes that extend RpcServer. + * + * @param req a Request + */ + @SuppressWarnings({"UnusedDeclaration"}) + public final void getConfigV3(Request req) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "getConfigV3"); + } + req.detach(); + JRTServerConfigRequestV3 request = JRTServerConfigRequestV3.createFromRequest(req); + addToRequestQueue(request); + } + + /** + * Returns 0 if server is alive. + * + * @param req a Request + */ + @SuppressWarnings("UnusedDeclaration") + public final void ping(Request req) { + req.returnValues().add(new Int32Value(0)); + } + + /** + * Returns a String with statistics data for the server. + * + * @param req a Request + */ + public final void printStatistics(Request req) { + req.returnValues().add(new StringValue("Delayed responses queue size: " + delayedConfigResponses.size())); + } + + public void run() { + log.log(LogLevel.DEBUG, "Ready for requests on " + spec); + try { + Acceptor acceptor = supervisor.listen(spec); + running = true; + supervisor.transport().join(); + acceptor.shutdown().join(); + } catch (ListenFailedException e) { + stop(); + throw new RuntimeException("Could not listen at " + spec, e); + } + running = false; + } + + public void stop() { + executorService.shutdown(); + try { + executorService.awaitTermination(SHUTDOWN_TIMEOUT, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.interrupted(); // Ignore and continue shutdown. + } + delayedConfigResponses.stop(); + supervisor.transport().shutdown().join(); + } + + /** + * Set up RPC method handlers. + */ + private void setUpHandlers() { + // The getConfig method in this class will handle RPC calls for getting config + getSupervisor().addMethod(JRTMethods.createConfigV3GetConfigMethod(this, "getConfigV3")); + getSupervisor().addMethod(new Method("ping", "", "i", + this, "ping") + .methodDesc("ping") + .returnDesc(0, "ret code", "return code, 0 is OK")); + getSupervisor().addMethod(new Method("printStatistics", "", "s", + this, "printStatistics") + .methodDesc("printStatistics") + .returnDesc(0, "statistics", "Statistics for server")); + } + + public boolean isRunning() { + return running; + } + + /** + * Checks all delayed responses for config changes and waits until all has been answered. + * This method should be called when config is reloaded in the server. + */ + @Override + public void configReloaded(TenantName tenant, ApplicationSet applicationSet) { + final ApplicationId applicationId = applicationSet.getId(); + configReloaded(delayedConfigResponses.drainQueue(applicationId), Tenants.logPre(applicationId)); + reloadSuperModel(tenant, applicationSet); + } + + private void reloadSuperModel(TenantName tenant, ApplicationSet applicationSet) { + superModelController.reloadConfig(tenant, applicationSet); + configReloaded(delayedConfigResponses.drainQueue(ApplicationId.global()), Tenants.logPre(ApplicationId.global())); + } + + private void configReloaded(List<DelayedConfigResponses.DelayedConfigResponse> responses, String logPre) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue"); + } + int responsesSent = 0; + CompletionService<Boolean> completionService = new ExecutorCompletionService<>(executorService); + while (!responses.isEmpty()) { + DelayedConfigResponses.DelayedConfigResponse delayedConfigResponse = responses.remove(0); + // Discard the ones that we have already answered + // Doing cancel here deals with the case where the timer is already running or has not run, so + // there is no need for any extra check. + if (delayedConfigResponse.cancel()) { + if (log.isLoggable(LogLevel.DEBUG)) { + logRequestDebug(LogLevel.DEBUG, logPre + "Timer cancelled for ", delayedConfigResponse.request); + } + // Do not wait for this request if we were unable to execute + if (addToRequestQueue(delayedConfigResponse.request, false, completionService)) { + responsesSent++; + } + } else { + log.log(LogLevel.DEBUG, logPre + "Timer already cancelled or finished or never scheduled"); + } + } + + for (int i = 0; i < responsesSent; i++) { + + try { + completionService.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + log.log(LogLevel.DEBUG, logPre + "Finished reloading " + responsesSent + " requests"); + } + + private void logRequestDebug(LogLevel level, String message, JRTServerConfigRequest request) { + if (log.isLoggable(level)) { + log.log(level, message + request.getShortDescription()); + } + } + + @Override + public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { + log.log(LogLevel.DEBUG, "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); + hostRegistry.update(tenant, newHosts); + } + + @Override + public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { + hostRegistry.verifyHosts(tenant, newHosts); + } + + @Override + public void applicationRemoved(ApplicationId applicationId) { + superModelController.removeApplication(applicationId); + configReloaded(delayedConfigResponses.drainQueue(applicationId), Tenants.logPre(applicationId)); + configReloaded(delayedConfigResponses.drainQueue(ApplicationId.global()), Tenants.logPre(ApplicationId.global())); + } + + public Spec getSpec() { + return spec; + } + + public void respond(JRTServerConfigRequest request) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Trace at request return:\n" + request.getRequestTrace().toString()); + } + request.getRequest().returnRequest(); + } + + /** + * Returns the tenant for this request, empty if there is no tenant for this request + * (which on hosted Vespa means that the requesting host is not currently active for any tenant) + */ + public Optional<TenantName> resolveTenant(JRTServerConfigRequest request, Trace trace) { + if ("*".equals(request.getConfigKey().getConfigId())) return Optional.of(ApplicationId.global().tenant()); + String hostname = request.getClientHostName(); + TenantName tenant = hostRegistry.getKeyForHost(hostname); + if (tenant == null) { + if (GetConfigProcessor.logDebug(trace)) { + String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName(); + log.log(LogLevel.DEBUG, message); + log.log(LogLevel.DEBUG, "hosts in host registry: " + hostRegistry.getAllHosts()); + trace.trace(6, message); + } + return Optional.empty(); + } + return Optional.of(tenant); + } + + public ConfigResponse resolveConfig(JRTServerConfigRequest request, GetConfigContext context, Optional<Version> vespaVersion) { + Trace trace = context.trace(); + if (trace.shouldTrace(TRACELEVEL)) { + trace.trace(TRACELEVEL, "RpcServer.resolveConfig()"); + } + RequestHandler handler = context.requestHandler(); + return handler.resolveConfig(context.applicationId(), request, vespaVersion); + + } + + protected Supervisor getSupervisor() { + return supervisor; + } + + Boolean addToRequestQueue(JRTServerConfigRequest request) { + return addToRequestQueue(request, false, null); + } + + public Boolean addToRequestQueue(JRTServerConfigRequest request, boolean forceResponse, CompletionService<Boolean> completionService) { + // It's no longer delayed if we get here + request.setDelayedResponse(false); + //ConfigDebug.logDebug(log, System.currentTimeMillis(), request.getConfigKey(), "RpcServer.addToRequestQueue()"); + try { + final GetConfigProcessor task = new GetConfigProcessor(this, request, forceResponse); + if (completionService == null) { + executorService.submit(task); + } else { + completionService.submit(new Callable<Boolean>() { + @Override + public Boolean call() throws Exception { + task.run(); + return true; + } + }); + } + updateWorkQueueMetrics(); + return true; + } catch (RejectedExecutionException e) { + request.addErrorResponse(ErrorCode.INTERNAL_ERROR, "getConfig request queue size is larger than configured max limit"); + respond(request); + return false; + } + } + + private void updateWorkQueueMetrics() { + int queued = executorService.getQueue().size(); + metrics.setRpcServerQueueSize(queued); + } + + /** + * Returns the context for this request, or null if the server is not properly set up with handlers + */ + public GetConfigContext createGetConfigContext(Optional<TenantName> optionalTenant, JRTServerConfigRequest request, Trace trace) { + if ("*".equals(request.getConfigKey().getConfigId())) { + return GetConfigContext.create(ApplicationId.global(), superModelController, trace); + } + TenantName tenant = optionalTenant.orElse(TenantName.defaultName()); // perhaps needed for non-hosted? + if ( ! hasRequestHandler(tenant)) { + String msg = Tenants.logPre(tenant) + "Unable to find request handler for tenant. Requested from host '" + request.getClientHostName() + "'"; + metrics.incUnknownHostRequests(); + trace.trace(TRACELEVEL, msg); + log.log(LogLevel.WARNING, msg); + return null; + } + RequestHandler handler = getRequestHandler(tenant); + ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName()); + if (trace.shouldTrace(TRACELEVEL_DEBUG)) { + trace.trace(TRACELEVEL_DEBUG, "Host '" + request.getClientHostName() + "' should have config from application '" + applicationId + "'"); + } + return GetConfigContext.create(applicationId, handler, trace); + } + + private boolean hasRequestHandler(TenantName tenant) { + return tenantProviders.containsKey(tenant); + } + + private RequestHandler getRequestHandler(TenantName tenant) { + if (!tenantProviders.containsKey(tenant)) { + throw new IllegalStateException("No request handler for " + tenant); + } + return tenantProviders.get(tenant).getRequestHandler(); + } + + public void delayResponse(JRTServerConfigRequest request, GetConfigContext context) { + delayedConfigResponses.delayResponse(request, context); + } + + @Override + public void onTenantDelete(TenantName tenant) { + log.log(LogLevel.DEBUG, Tenants.logPre(tenant)+"Tenant deleted, removing request handler and cleaning host registry"); + if (tenantProviders.containsKey(tenant)) { + tenantProviders.remove(tenant); + } + hostRegistry.removeHostsForKey(tenant); + } + + @Override + public void onTenantsLoaded() { + allTenantsLoaded = true; + superModelController.enable(); + } + + @Override + public void onTenantCreate(TenantName tenant, TenantHandlerProvider tenantHandlerProvider) { + log.log(LogLevel.DEBUG, Tenants.logPre(tenant)+"Tenant created, adding request handler"); + tenantProviders.put(tenant, tenantHandlerProvider); + } + + /** Returns true only after all tenants are loaded */ + public boolean allTenantsLoaded() { return allTenantsLoaded; } + + /** Returns true if this rpc server is currently running in a hosted Vespa configuration */ + public boolean isHostedVespa() { return hostedVespa; } + + MetricUpdaterFactory metricUpdaterFactory() { + return metricUpdaterFactory; + } + + boolean useRequestVersion() { + return useRequestVersion; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java new file mode 100644 index 00000000000..8858d368edf --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.config.ConfigCacheKey; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.protocol.ConfigResponse; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Cache that holds configs and config definitions. It has separate maps for the separate + * "types", for clarity. + * + * @author vegardh + */ +public class ServerCache { + + private final Map<ConfigDefinitionKey, ConfigDefinition> defs = new ConcurrentHashMap<>(); + /* Legacy user configs from configs/ dir in application package (NB! Only name, not key) */ + private final Map<String, ConfigPayload> legacyUserCfgs = new ConcurrentHashMap<>(); + + // NOTE: The reason we do a double mapping here is to dedup configs that have the same md5. + private final Map<ConfigCacheKey, String> md5Sums = new ConcurrentHashMap<>(); + private final Map<String, ConfigResponse> md5ToConfig = new ConcurrentHashMap<>(); + + public void addDef(ConfigDefinitionKey key, ConfigDefinition def) { + defs.put(key, def); + } + + public void addLegacyUserConfig(String name, ConfigPayload config) { + legacyUserCfgs.put(name, config); + } + + public ConfigPayload getLegacyUserConfig(String name) { + return legacyUserCfgs.get(name); + } + + public void put(ConfigCacheKey key, ConfigResponse config, String configMd5) { + md5Sums.put(key, configMd5); + md5ToConfig.put(configMd5, config); + } + + public ConfigResponse get(ConfigCacheKey key) { + String md5 = md5Sums.get(key); + if (md5 == null) return null; + return md5ToConfig.get(md5); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("Cache\n"); + sb.append("defs: ").append(defs.size()).append("\n"); + sb.append("user cfgs: ").append(legacyUserCfgs.size()).append("\n"); + sb.append("md5sums: ").append(md5Sums.size()).append("\n"); + sb.append("md5ToConfig: ").append(md5ToConfig.size()).append("\n"); + + return sb.toString(); + } + + public ConfigDefinition getDef(ConfigDefinitionKey defKey) { + return defs.get(defKey); + } + + /** + * The number of different {@link ConfigResponse} elements + * @return elems + */ + public int configElems() { + return md5ToConfig.size(); + } + + /** + * The number of different key→checksum mappings + * @return elems + */ + public int checkSumElems() { + return md5Sums.size(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java new file mode 100644 index 00000000000..b3921da869a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/StaticConfigDefinitionRepo.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; + +import java.util.Collections; +import java.util.Map; + +/** + * A global pool of all config definitions that this server knows about. These objects can be shared + * by all tenants, as they are not modified. + * + * @author lulf + * @since 5.10 + */ +public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { + + private final ConfigDefinitionRepo repo; + + // Only useful in tests that dont need full blown repo. + public StaticConfigDefinitionRepo() { + this.repo = new ConfigDefinitionRepo() { + @Override + public Map<ConfigDefinitionKey, ConfigDefinition> getConfigDefinitions() { + return Collections.emptyMap(); + } + }; + } + + @Inject + public StaticConfigDefinitionRepo(ConfigServerDB serverDB) { + this.repo = new com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo(serverDB.serverdefs()); + } + + @Override + public Map<ConfigDefinitionKey, ConfigDefinition> getConfigDefinitions() { + return repo.getConfigDefinitions(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java new file mode 100644 index 00000000000..f6b00440e30 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.config.server.model.SuperModel; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import com.yahoo.cloud.config.ElkConfig; + +/** + * Controls the lifetime of the {@link SuperModel} and the {@link SuperModelRequestHandler}. + * + * @author lulf + * @since 5.9 + */ +public class SuperModelController implements RequestHandler { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SuperModelController.class.getName()); + private volatile SuperModelRequestHandler handler; + private final GenerationCounter generationCounter; + private final Zone zone; + private final long masterGeneration; + private final ConfigDefinitionRepo configDefinitionRepo; + private final ConfigResponseFactory responseFactory; + private final ElkConfig elkConfig; + private volatile boolean enabled = false; + + + public SuperModelController(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, ConfigserverConfig configserverConfig, ElkConfig elkConfig) { + this.generationCounter = generationCounter; + this.configDefinitionRepo = configDefinitionRepo; + this.elkConfig = elkConfig; + this.masterGeneration = configserverConfig.masterGeneration(); + this.responseFactory = ConfigResponseFactoryFactory.createFactory(configserverConfig); + this.zone = new Zone(configserverConfig); + this.handler = createNewHandler(Collections.emptyMap()); + } + + /** + * Signals that config has been reloaded for an {@link com.yahoo.vespa.config.server.application.Application} + * belonging to a tenant. + * + * TODO: This is a bit too complex I think. + * + * @param tenant Name of tenant owning the application. + * @param applicationSet The reloaded set of {@link com.yahoo.vespa.config.server.application.Application}. + */ + public synchronized void reloadConfig(TenantName tenant, ApplicationSet applicationSet) { + Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); + if (!newModels.containsKey(tenant)) { + newModels.put(tenant, new LinkedHashMap<>()); + } + // TODO: Should supermodel care about multiple versions? + newModels.get(tenant).put(applicationSet.getId(), applicationSet.getForVersionOrLatest(Optional.empty())); + handler = createNewHandler(newModels); + } + + public synchronized void removeApplication(ApplicationId applicationId) { + Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); + if (newModels.containsKey(applicationId.tenant())) { + newModels.get(applicationId.tenant()).remove(applicationId); + if (newModels.get(applicationId.tenant()).isEmpty()) { + newModels.remove(applicationId.tenant()); + } + } + handler = createNewHandler(newModels); + } + + private SuperModelRequestHandler createNewHandler(Map<TenantName, Map<ApplicationId, Application>> newModels) { + long generation = generationCounter.get() + masterGeneration; + SuperModel model = new SuperModel(newModels, elkConfig, zone); + return new SuperModelRequestHandler(model, configDefinitionRepo, generation, responseFactory); + } + + private Map<TenantName, Map<ApplicationId, Application>> getCurrentModels() { + if (handler != null) { + return handler.getSuperModel().getCurrentModels(); + } else { + return new LinkedHashMap<>(); + } + } + + private Map<TenantName, Map<ApplicationId, Application>> createModelCopy() { + Map<TenantName, Map<ApplicationId, Application>> currentModels = getCurrentModels(); + Map<TenantName, Map<ApplicationId, Application>> newModels = new LinkedHashMap<>(); + for (Map.Entry<TenantName, Map<ApplicationId, Application>> entry : currentModels.entrySet()) { + Map<ApplicationId, Application> appMap = new LinkedHashMap<>(); + newModels.put(entry.getKey(), appMap); + for (Map.Entry<ApplicationId, Application> appEntry : entry.getValue().entrySet()) { + appMap.put(appEntry.getKey(), appEntry.getValue()); + } + } + return newModels; + } + + public SuperModelRequestHandler getHandler() { return handler; } + + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + log.log(LogLevel.DEBUG, "SuperModelController resolving " + req + " for app id '" + appId + "'"); + if (handler != null) { + return handler.resolveConfig(req); + } + return null; + } + + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + return handler.getConfig(configClass, applicationId, configId); + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + throw new UnsupportedOperationException(); + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> key, boolean recursive) { + throw new UnsupportedOperationException(); + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + throw new UnsupportedOperationException(); + } + + @Override + public Set<String> allConfigIds(ApplicationId appID, Optional<Version> vespaVersion) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + return enabled && appId.equals(ApplicationId.global()); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + return ApplicationId.global(); + } + + public void enable() { + enabled = true; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java new file mode 100644 index 00000000000..589363467b0 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.path.Path; +import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.curator.recipes.CuratorCounter; +import com.yahoo.vespa.curator.Curator; + +/** + * Distributed global generation counter for the super model. + * + * @author lulf + * @since 5.9 + */ +public class SuperModelGenerationCounter implements GenerationCounter { + + private static final Path counterPath = Path.fromString("/config/v2/RPC/superModelGeneration"); + private final CuratorCounter counter; + + public SuperModelGenerationCounter(Curator curator) { + this.counter = new CuratorCounter(curator, counterPath.getAbsolute()); + } + + /** + * Increment counter and return next value. This method is thread safe and provides an atomic value + * across zookeeper clusters. + * + * @return incremented counter value. + */ + public synchronized long increment() { + return counter.next(); + } + + /** + * @return current counter value. + */ + public synchronized long get() { + return counter.get(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java new file mode 100644 index 00000000000..c745379f9cb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.server.model.SuperModel; + +import java.io.IOException; +import java.io.StringReader; + +/** + * Handler for global configs that must be resolved using the global SuperModel instance. Deals with + * reloading of config as well. + * + * @author lulf + * @since 5.9 + */ +public class SuperModelRequestHandler { + private final SuperModel model; + private final long generation; + private final ConfigDefinitionRepo configDefinitionRepo; + private final ConfigResponseFactory responseFactory; + + public SuperModelRequestHandler(SuperModel model, ConfigDefinitionRepo configDefinitionRepo, long generation, ConfigResponseFactory responseFactory) { + this.model = model; + this.configDefinitionRepo = configDefinitionRepo; + this.generation = generation; + this.responseFactory = responseFactory; + } + + /** + * Resolves global config for given request. + * + * @param request The {@link com.yahoo.vespa.config.GetConfigRequest} to find config for. + * @return a {@link com.yahoo.vespa.config.protocol.ConfigResponse} containing the response for this request. + * @throws java.lang.IllegalArgumentException if no such config was found. + */ + public ConfigResponse resolveConfig(GetConfigRequest request) { + ConfigKey<?> configKey = request.getConfigKey(); + InnerCNode targetDef = getConfigDefinition(request.getConfigKey(), request.getDefContent()); + try { + ConfigPayload payload = model.getConfig(configKey); + return responseFactory.createResponse(payload, targetDef, generation); + } catch (IOException e) { + throw new ConfigurationRuntimeException("Unable to resolve config", e); + } + } + + private InnerCNode getConfigDefinition(ConfigKey<?> configKey, DefContent defContent) { + if (defContent.isEmpty()) { + ConfigDefinitionKey configDefinitionKey = new ConfigDefinitionKey(configKey.getName(), configKey.getNamespace()); + ConfigDefinition configDefinition = configDefinitionRepo.getConfigDefinitions().get(configDefinitionKey); + if (configDefinition == null) { + throw new UnknownConfigDefinitionException("Unable to find config definition for '" + configKey.getNamespace() + "." + configKey.getName()); + } + return configDefinition.getCNode(); + } else { + DefParser dParser = new DefParser(configKey.getName(), new StringReader(defContent.asString())); + return dParser.getTree(); + } + } + + SuperModel getSuperModel() { + return model; + } + + long getGeneration() { return generation; } + + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + return model.getConfig(configClass, applicationId, configId); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/Tenant.java new file mode 100644 index 00000000000..cabf4bed323 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/Tenant.java @@ -0,0 +1,182 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.TenantName; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.RemoteSessionRepo; +import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; +import java.util.logging.Logger; + +/** + * Contains all tenant-level components for a single tenant, dealing with editing sessions and + * applications for a single tenant. + * + * @author vegardh + * @author lulf + * @since 5.1.26 + */ +public class Tenant implements TenantHandlerProvider { + + private static final Logger log = Logger.getLogger(Tenant.class.getName()); + static final String SESSIONS = "sessions"; + static final String APPLICATIONS = "applications"; + + private final TenantName name; + private final RemoteSessionRepo remoteSessionRepo; + private final Path path; + private final SessionFactory sessionFactory; + private final LocalSessionRepo localSessionRepo; + private final ApplicationRepo applicationRepo; + private final ActivateLock activateLock; + private final RequestHandler requestHandler; + private final ReloadHandler reloadHandler; + private final TenantFileSystemDirs tenantFileSystemDirs; + private final Curator curator; + + Tenant(TenantName name, + Path path, + SessionFactory sessionFactory, + LocalSessionRepo localSessionRepo, + RemoteSessionRepo remoteSessionRepo, + RequestHandler requestHandler, + ReloadHandler reloadHandler, + ApplicationRepo applicationRepo, + Curator curator, + TenantFileSystemDirs tenantFileSystemDirs) { + this.name = name; + this.path = path; + this.requestHandler = requestHandler; + this.reloadHandler = reloadHandler; + this.remoteSessionRepo = remoteSessionRepo; + this.sessionFactory = sessionFactory; + this.localSessionRepo = localSessionRepo; + this.activateLock = new ActivateLock(curator, path); + this.applicationRepo = applicationRepo; + this.tenantFileSystemDirs = tenantFileSystemDirs; + this.curator = curator; + } + + /** + * The reload handler for this + * + * @return handler + */ + public ReloadHandler getReloadHandler() { + return reloadHandler; + } + + /** + * The request handler for this + * + * @return handler + */ + public RequestHandler getRequestHandler() { + return requestHandler; + } + + /** + * The RemoteSessionRepo for this + * + * @return repo + */ + public RemoteSessionRepo getRemoteSessionRepo() { + return remoteSessionRepo; + } + + public TenantName getName() { + return name; + } + + public Path getPath() { + return path; + } + + public SessionFactory getSessionFactory() { + return sessionFactory; + } + + public LocalSessionRepo getLocalSessionRepo() { + return localSessionRepo; + } + + /** + * The activation lock for this + * @return lock + */ + public ActivateLock getActivateLock() { + return activateLock; + } + + @Override + public String toString() { + return getName().value(); + } + + public ApplicationRepo getApplicationRepo() { + return applicationRepo; + } + + public Curator getCurator() { + return curator; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Tenant)) { + return false; + } + Tenant that = (Tenant) other; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + /** + * Closes any watchers, thread pools that may react to changes in tenant state. Also removes any local state + * in filesystem. + */ + public void close() { + tenantFileSystemDirs.delete(); + remoteSessionRepo.close(); + applicationRepo.close(); + } + + /** + * Deletes a tenant from ZooKeeper and filesystem. + */ + public void delete() { + localSessionRepo.deleteAllSessions(); + curator.delete(path); + } + + public void redeployApplications(Deployer deployer) { + // TODO: Configurable timeout + applicationRepo.listApplications().stream() + .forEach(applicationId -> redeployApplication(applicationId, deployer)); + } + + private void redeployApplication(ApplicationId applicationId, Deployer deployer) { + try { + log.log(LogLevel.DEBUG, "Redeploying " + applicationId); + deployer.deployFromLocalActive(applicationId, Duration.ofMinutes(30)) + .ifPresent(deployment -> { + deployment.prepare(); + deployment.activate(); + }); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Redeploying " + applicationId + " failed", e); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantBuilder.java new file mode 100644 index 00000000000..78ec368f2fb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantBuilder.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.path.Path; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.application.ZKApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.session.*; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.time.Clock; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Builder for helping out with tenant creation. Each of a tenants dependencies may be overridden for testing. + * + * @author lulf + * @since 5.1 + */ +public class TenantBuilder { + private final Path tenantPath; + private final GlobalComponentRegistry componentRegistry; + private final TenantName tenant; + private final Path sessionsPath; + private RemoteSessionRepo remoteSessionRepo; + private LocalSessionRepo localSessionRepo; + private SessionFactory sessionFactory; + private LocalSessionLoader localSessionLoader; + private ApplicationRepo applicationRepo; + private SessionCounter sessionCounter; + private ReloadHandler reloadHandler; + private RequestHandler requestHandler; + private RemoteSessionFactory remoteSessionFactory; + private TenantFileSystemDirs tenantFileSystemDirs; + private HostValidator<ApplicationId> hostValidator; + + private TenantBuilder(GlobalComponentRegistry componentRegistry, TenantName tenant, Path zkPath) { + this.componentRegistry = componentRegistry; + this.tenantPath = zkPath; + this.tenant = tenant; + this.sessionsPath = tenantPath.append(Tenant.SESSIONS); + } + + public static TenantBuilder create(GlobalComponentRegistry componentRegistry, TenantName tenant, Path zkPath) { + return new TenantBuilder(componentRegistry, tenant, zkPath); + } + + public TenantBuilder withSessionFactory(SessionFactory sessionFactory) { + this.sessionFactory = sessionFactory; + return this; + } + + public TenantBuilder withLocalSessionRepo(LocalSessionRepo localSessionRepo) { + this.localSessionRepo = localSessionRepo; + return this; + } + + public TenantBuilder withRemoteSessionRepo(RemoteSessionRepo remoteSessionRepo) { + this.remoteSessionRepo = remoteSessionRepo; + return this; + } + + public TenantBuilder withApplicationRepo(ApplicationRepo applicationRepo) { + this.applicationRepo = applicationRepo; + return this; + } + + public TenantBuilder withRequestHandler(RequestHandler requestHandler) { + this.requestHandler = requestHandler; + return this; + } + + public TenantBuilder withReloadHandler(ReloadHandler reloadHandler) { + this.reloadHandler = reloadHandler; + return this; + } + + /** + * Create a real tenant from the properties given by this builder. + * + * @return a new {@link Tenant} instance. + * @throws Exception if building fails + */ + public Tenant build() throws Exception { + createTenantRequestHandler(); + createApplicationRepo(); + createRemoteSessionFactory(); + createRemoteSessionRepo(); + createSessionCounter(); + createServerDbDirs(); + createSessionFactory(); + createLocalSessionRepo(); + return new Tenant(tenant, + tenantPath, + sessionFactory, + localSessionRepo, + remoteSessionRepo, + requestHandler, + reloadHandler, + applicationRepo, + componentRegistry.getCurator(), + tenantFileSystemDirs); + } + + private void createLocalSessionRepo() { + if (localSessionRepo == null) { + localSessionRepo = new LocalSessionRepo(tenantFileSystemDirs, localSessionLoader, applicationRepo, Clock.systemUTC(), componentRegistry.getConfigserverConfig().sessionLifetime()); + } + } + + private void createSessionFactory() { + if (sessionFactory == null || localSessionLoader == null) { + SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, sessionCounter, sessionsPath, applicationRepo, tenantFileSystemDirs, hostValidator, tenant); + if (sessionFactory == null) { + sessionFactory = impl; + } + if (localSessionLoader == null) { + localSessionLoader = impl; + } + } + } + + private void createApplicationRepo() { + if (applicationRepo == null) { + applicationRepo = ZKApplicationRepo.create(componentRegistry.getCurator(), tenantPath.append(Tenant.APPLICATIONS), reloadHandler, tenant); + } + } + + private void createSessionCounter() { + if (sessionCounter == null) { + sessionCounter = new SessionCounter(componentRegistry.getCurator(), tenantPath, sessionsPath); + } + } + + private void createTenantRequestHandler() { + if (requestHandler == null || reloadHandler == null) { + TenantRequestHandler impl = new TenantRequestHandler(componentRegistry.getMetrics(), + tenant, + Collections.singletonList(componentRegistry.getReloadListener()), + ConfigResponseFactoryFactory.createFactory(componentRegistry.getConfigserverConfig()), + componentRegistry.getHostRegistries()); + if (hostValidator == null) { + this.hostValidator = impl; + } + if (requestHandler == null) { + requestHandler = impl; + } + if (reloadHandler == null) { + reloadHandler = impl; + } + } + } + + private void createRemoteSessionFactory() { + if (remoteSessionFactory == null) { + remoteSessionFactory = new RemoteSessionFactory( + componentRegistry, + sessionsPath, + tenant); + } + } + + private void createRemoteSessionRepo() throws Exception { + if (remoteSessionRepo == null) { + remoteSessionRepo = RemoteSessionRepo.create(componentRegistry.getCurator(), + remoteSessionFactory, + reloadHandler, + sessionsPath, + applicationRepo, + componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant)), + createSingleThreadedExecutorService(RemoteSessionRepo.class.getName())); + } + } + + private ExecutorService createSingleThreadedExecutorService(String executorName) { + return Executors.newSingleThreadExecutor(ThreadFactoryFactory.getThreadFactory(executorName + "-" + tenant.value())); + } + + private void createServerDbDirs() { + if (tenantFileSystemDirs == null) { + tenantFileSystemDirs = new TenantFileSystemDirs(new File(Defaults.getDefaults().underVespaHome(componentRegistry.getServerDB().getConfigserverConfig().configServerDBDir())), tenant); + } + } + + + public LocalSessionRepo getLocalSessionRepo() { + return localSessionRepo; + } + + public ApplicationRepo getApplicationRepo() { + return applicationRepo; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TenantDebugger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantDebugger.java new file mode 100644 index 00000000000..ab5bf6db7b9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantDebugger.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Curator; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.TreeCache; +import org.apache.curator.framework.recipes.cache.TreeCacheEvent; +import org.apache.curator.framework.recipes.cache.TreeCacheListener; + +import java.util.logging.Logger; + +/** + * For debugging tenant issues in configserver. Activate by loading component. + * + * @author lulf + */ +public class TenantDebugger implements TreeCacheListener { + private final TreeCache cache; + private static final Logger log = Logger.getLogger(TenantDebugger.class.getName()); + + public TenantDebugger(Curator curator) throws Exception { + cache = new TreeCache(curator.framework(), "/config/v2/tenants"); + cache.getListenable().addListener(this); + cache.start(); + } + + @Override + public void childEvent(CuratorFramework client, TreeCacheEvent event) throws Exception { + switch (event.getType()) { + case NODE_ADDED: + case NODE_REMOVED: + case NODE_UPDATED: + log.log(LogLevel.INFO, event.toString()); + break; + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TenantHandlerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantHandlerProvider.java new file mode 100644 index 00000000000..a9321709dd3 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantHandlerProvider.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +/** + * Represents something that can provide request and reload handlers of a tenant. + * + * @author lulf + * @since 5.3 + */ +public interface TenantHandlerProvider { + + RequestHandler getRequestHandler(); + ReloadHandler getReloadHandler(); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TenantListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantListener.java new file mode 100644 index 00000000000..7037e24ff5d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantListener.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.TenantName; + +/** + * Interface for something that listens for created and deleted tenants. + * + * @author lulf + * @since 5.8 + */ +public interface TenantListener { + /** + * Called whenever a new tenant is created. + * + * @param tenant name of newly created tenant. + * @param provider provider of request and reload handlers for new tenant. + */ + public void onTenantCreate(TenantName tenant, TenantHandlerProvider provider); + + /** + * Called whenever a tenant is deleted. + * + * @param tenant name of deleted tenant. + */ + public void onTenantDelete(TenantName tenant); + + /** + * Called when all tenants have been loaded at startup. + */ + void onTenantsLoaded(); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantRequestHandler.java new file mode 100644 index 00000000000..c4e77abc6e9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TenantRequestHandler.java @@ -0,0 +1,213 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.*; + +import com.yahoo.config.provision.Version; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; + +/** + * A per tenant request handler, for handling reload (activate application) and getConfig requests for + * a set of applications belonging to a tenant. + * + * @author Harald Musum + * @since 5.1 + */ +public class TenantRequestHandler implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(TenantRequestHandler.class.getName()); + + private final Metrics metrics; + private final TenantName tenant; + private final List<ReloadListener> reloadListeners; + private final ConfigResponseFactory responseFactory; + + private final HostRegistry<ApplicationId> hostRegistry; + private final ApplicationMapper applicationMapper = new ApplicationMapper(); + private final MetricUpdater tenantMetricUpdater; + + public TenantRequestHandler(Metrics metrics, + TenantName tenant, + List<ReloadListener> reloadListeners, + ConfigResponseFactory responseFactory, + HostRegistries hostRegistries) { + this.metrics = metrics; + this.tenant = tenant; + this.reloadListeners = reloadListeners; + this.responseFactory = responseFactory; + tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); + hostRegistry = hostRegistries.createApplicationHostRegistry(tenant); + } + + /** + * Gets a config for the given app, or null if not found + */ + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, Tenants.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); + } + return application.resolveConfig(req, responseFactory); + } + + // For testing only + long getApplicationGeneration(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.getApplicationGeneration(); + } + + private void notifyReloadListeners(ApplicationSet applicationSet) { + for (ReloadListener reloadListener : reloadListeners) { + reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + reloadListener.configReloaded(tenant, applicationSet); + } + } + + /** + * Activates the config of the given app. Notifies listeners + * @param applicationSet the {@link com.yahoo.vespa.config.server.ApplicationSet} to be reloaded + */ + public void reloadConfig(ApplicationSet applicationSet) { + setLiveApp(applicationSet); + notifyReloadListeners(applicationSet); + } + + @Override + public void removeApplication(ApplicationId applicationId) { + if (applicationMapper.hasApplication(applicationId)) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + } + } + + private void reloadListenersOnRemove(ApplicationId applicationId) { + for (ReloadListener listener : reloadListeners) { + listener.applicationRemoved(applicationId); + listener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + } + } + + private void setLiveApp(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + final Collection<String> hostsForApp = applicationSet.getAllHosts(); + hostRegistry.update(id, hostsForApp); + applicationSet.updateHostMetrics(); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + applicationMapper.register(id, applicationSet); + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, keyToMatch, recursive); + } + + private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> key : application.allConfigsProduced()) { + String configId = key.getConfigId(); + if (recursive) { + key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); + } else { + // Include first part of id as id + key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); + } + if (keyToMatch != null) { + String n = key.getName(); // Never null + String ns = key.getNamespace(); // Never null + if (n.equals(keyToMatch.getName()) && + ns.equals(keyToMatch.getNamespace()) && + configId.startsWith(keyToMatch.getConfigId()) && + !(configId.equals(keyToMatch.getConfigId()))) { + + if (!recursive) { + // For non-recursive, include the id segment we were searching for, and first part of the rest + key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); + } + ret.add(key); + } + } else { + ret.add(key); + } + } + return ret; + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, null, recursive); + } + + /** + * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers + * @return id segment with one extra level from the id appended + */ + String appendOneLevelOfId(String baseIdSegment, String id) { + if ("".equals(baseIdSegment)) return id.split("/")[0]; + String theRest = id.substring(baseIdSegment.length()); + if ("".equals(theRest)) return id; + theRest = theRest.replaceFirst("/", ""); + String theRestFirstSeg = theRest.split("/")[0]; + return baseIdSegment+"/"+theRestFirstSeg; + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigsProduced(); + } + + private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { + try { + return applicationMapper.getForVersion(appId, vespaVersion); + } catch (VersionDoesNotExistException ex) { + throw new NotFoundException(String.format("%sNo such application (id %s): %s", Tenants.logPre(tenant), appId, ex.getMessage())); + } + } + + @Override + public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigIds(); + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + return hasHandler(appId, vespaVersion); + } + + private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { + return applicationMapper.hasApplicationForVersion(appId, vespaVersion); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); + if (applicationId == null) { + applicationId = ApplicationId.defaultId(); + } + return applicationId; + } + + @Override + public void verifyHosts(ApplicationId key, Collection<String> newHosts) { + hostRegistry.verifyHosts(key, newHosts); + for (ReloadListener reloadListener : reloadListeners) { + reloadListener.verifyHostsAreAvailable(tenant, newHosts); + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/Tenants.java new file mode 100644 index 00000000000..63ae8d1f743 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/Tenants.java @@ -0,0 +1,358 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.inject.Inject; +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.TenantName; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.curator.Curator; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.framework.state.ConnectionStateListener; +import org.apache.zookeeper.KeeperException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * This component will monitor the set of tenants in the config server by watching in ZooKeeper. + * It will set up Tenant objects accordingly, which will manage the config sessions per tenant. + * This class will read the preexisting set of tenants from ZooKeeper at startup. (For now it will also + * create a default tenant since that will be used for API that do no know about tenants or have not yet + * implemented support for it). + * + * This instance is called from two different threads, the http handler threads and the zookeeper watcher threads. + * To create or delete a tenant, the handler calls {@link Tenants#createTenant} and {@link Tenants#deleteTenant} methods. + * This will delete shared state from zookeeper, and return, so it does not mean a tenant is immediately deleted. + * + * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all configservers, and + * shutdown and delete any per-configserver state. + * + * @author vegardh + * @author lulf + * @since 5.1.26 + */ +public class Tenants implements ConnectionStateListener, PathChildrenCacheListener { + + private static final Logger log = Logger.getLogger(Tenants.class.getName()); + + private static final TenantName DEFAULT_TENANT = TenantName.defaultName(); + private static final List<TenantName> SYSTEM_TENANT_NAMES = Arrays.asList( + DEFAULT_TENANT, + ApplicationId.HOSTED_VESPA_TENANT); + private static final Path tenantsPath = Path.fromString("/config/v2/tenants/"); + private static final Path vespaPath = Path.fromString("/vespa"); + + private final Map<TenantName, Tenant> tenants = new LinkedHashMap<>(); + private final GlobalComponentRegistry globalComponentRegistry; + private final List<TenantListener> tenantListeners = Collections.synchronizedList(new ArrayList<>()); + private final Curator curator; + + private final MetricUpdater metricUpdater; + private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(Tenants.class.getName())); + private final Curator.DirectoryCache directoryCache; + + + /** + * New instance from the tenants in the given component registry's ZooKeeper. Will set watch when reading them. + * + * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @throws Exception is creating the Tenants instance fails + */ + @Inject + public Tenants(GlobalComponentRegistry globalComponentRegistry, Metrics metrics) throws Exception { + // Note: unit tests may want to use the constructor below to avoid setting watch by calling readTenants(). + this.globalComponentRegistry = globalComponentRegistry; + this.curator = globalComponentRegistry.getCurator(); + metricUpdater = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); + this.tenantListeners.add(globalComponentRegistry.getTenantListener()); + curator.framework().getConnectionStateListenable().addListener(this); + + curator.create(tenantsPath); + createSystemTenants(); + curator.create(vespaPath); + + this.directoryCache = globalComponentRegistry.getCurator().createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); + directoryCache.start(); + directoryCache.addListener(this); + tenantsChanged(readTenants()); + notifyTenantsLoaded(); + } + + private void notifyTenantsLoaded() { + for (TenantListener tenantListener : tenantListeners) { + tenantListener.onTenantsLoaded(); + } + } + + /** + * New instance containing the given tenants. This will not watch in ZooKeeper. + * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} instance + * @param metrics a {@link com.yahoo.vespa.config.server.monitoring.Metrics} instance + * @param tenants a collection of {@link Tenant}s + */ + public Tenants(GlobalComponentRegistry globalComponentRegistry, Metrics metrics, Collection<Tenant> tenants) { + this.globalComponentRegistry = globalComponentRegistry; + this.curator = globalComponentRegistry.getCurator(); + metricUpdater = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); + this.tenantListeners.add(globalComponentRegistry.getTenantListener()); + curator.create(tenantsPath); + this.directoryCache = curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor); + this.tenants.putAll(addTenants(tenants)); + } + + // Pre-condition: tenants path needs to exist in zk + private LinkedHashMap<TenantName, Tenant> addTenants(Collection<Tenant> newTenants) { + final LinkedHashMap<TenantName, Tenant> sessionTenants = new LinkedHashMap<>(); + for (Tenant t : newTenants) { + sessionTenants.put(t.getName(), t); + } + log.log(LogLevel.DEBUG, "Tenants at startup: " + sessionTenants); + metricUpdater.setTenants(tenants.size()); + return sessionTenants; + } + + /** + * Reads the set of tenants in patch cache. + * + * @return a set of tenant names + */ + private Set<TenantName> readTenants() { + Set<TenantName> tenants = new LinkedHashSet<>(); + for (String tenant : curator.getChildren(tenantsPath)) { + tenants.add(TenantName.from(tenant)); + } + return tenants; + } + + synchronized void tenantsChanged(Set<TenantName> newTenants) throws Exception { + log.log(LogLevel.DEBUG, "Tenants changed: " + newTenants); + checkForRemovedTenants(newTenants); + checkForAddedTenants(newTenants); + metricUpdater.setTenants(tenants.size()); + } + + private void checkForRemovedTenants(Set<TenantName> newTenants) { + Map<TenantName, Tenant> current = new LinkedHashMap<>(tenants); + for (Map.Entry<TenantName, Tenant> entry : current.entrySet()) { + TenantName tenant = entry.getKey(); + if (!newTenants.contains(tenant)) { + notifyRemovedTenant(tenant); + entry.getValue().close(); + tenants.remove(tenant); + } + } + } + + private void checkForAddedTenants(Set<TenantName> newTenants) + throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(globalComponentRegistry.getConfigserverConfig().numParallelTenantLoaders()); + Map<TenantName, Tenant> addedTenants = new ConcurrentHashMap<>(); + for (TenantName tenantName : newTenants) { + // Note: the http handler will check if the tenant exists, and throw accordingly + if (!tenants.containsKey(tenantName)) { + executor.execute(() -> { + try { + Tenant tenant = TenantBuilder.create(globalComponentRegistry, tenantName, getTenantPath(tenantName)).build(); + notifyNewTenant(tenant); + addedTenants.put(tenantName, tenant); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Error loading tenant '" + tenantName + "', skipping.", e); + } + }); + } + } + executor.shutdown(); + executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen + tenants.putAll(addedTenants); + } + + /** + * The registered tenants. Creates a copy of the map to avoid it being modified outside, since it can + * change after this method has been called. + * + * @return tenant list + */ + public synchronized Map<TenantName, Tenant> tenantsCopy() { + return new LinkedHashMap<>(tenants); + } + + /** + * Returns a default (compatibility with single tenant config requests) tenant + * + * @return default tenant + */ + public synchronized Tenant defaultTenant() { + return tenants.get(DEFAULT_TENANT); + } + + private void notifyNewTenant(Tenant tenant) { + for (TenantListener listener : tenantListeners) { + listener.onTenantCreate(tenant.getName(), tenant); + } + } + + private void notifyRemovedTenant(TenantName name) { + for (TenantListener listener : tenantListeners) { + listener.onTenantDelete(name); + } + } + + /** + * Writes the default tenant into ZooKeeper. Will not fail if the node already exists, + * as this is OK and might happen when several config servers start at the same time and + * try to call this method. + */ + public synchronized void createSystemTenants() { + for (final TenantName tenantName : SYSTEM_TENANT_NAMES) { + try { + createTenant(tenantName); + } catch (RuntimeException e) { + // Do nothing if we get NodeExistsException + if (e.getCause().getClass() != KeeperException.NodeExistsException.class) { + throw e; + } + } + } + } + + /** + * Writes the given tenant into ZooKeeper, for watchers to react on + * + * @param name name of the tenant + * @return this Tenants + */ + public synchronized Tenants createTenant(TenantName name) { + Path tenantPath = getTenantPath(name); + curator.createAtomically(tenantPath, tenantPath.append(Tenant.SESSIONS), tenantPath.append(Tenant.APPLICATIONS)); + return this; + } + + /** + * Removes the given tenant from ZooKeeper and filesystem. Assumes that tenant exists. + * + * @param name name of the tenant + * @return this Tenants instance + */ + public synchronized Tenants deleteTenant(TenantName name) { + Tenant tenant = tenants.get(name); + tenant.delete(); + return this; + } + + // For unit testing + String tenantZkPath(TenantName tenant) { + return getTenantPath(tenant).getAbsolute(); + } + + /** + * A helper to format a log preamble for messages with a tenant and app id + * @param app the app + * @return the log string + */ + public static String logPre(ApplicationId app) { + if (TenantName.defaultName().equals(app.tenant())) return ""; + StringBuilder ret = new StringBuilder() + .append(logPre(app.tenant())) + .append("app:"+app.application().value()) + .append(":"+app.instance().value()) + .append(" "); + return ret.toString(); + } + + /** + * A helper to format a log preamble for messages with a tenant + * @param tenant tenant + * @return the log string + */ + public static String logPre(TenantName tenant) { + if (DEFAULT_TENANT.equals(tenant)) return ""; + StringBuilder ret = new StringBuilder() + .append("tenant:"+tenant.value()) + .append(" "); + return ret.toString(); + } + + @Override + public void stateChanged(CuratorFramework framework, ConnectionState connectionState) { + switch (connectionState) { + case CONNECTED: + metricUpdater.incZKConnected(); + break; + case SUSPENDED: + metricUpdater.incZKSuspended(); + break; + case RECONNECTED: + metricUpdater.incZKReconnected(); + break; + case LOST: + metricUpdater.incZKConnectionLost(); + break; + case READ_ONLY: + // NOTE: Should not be relevant for configserver. + break; + } + } + + @Override + public void childEvent(CuratorFramework framework, PathChildrenCacheEvent event) throws Exception { + switch (event.getType()) { + case CHILD_ADDED: + case CHILD_REMOVED: + tenantsChanged(readTenants()); + break; + } + } + + public void close() throws IOException { + directoryCache.close(); + pathChildrenExecutor.shutdown(); + } + + public void redeployApplications(Deployer deployer) { + final int totalNumberOfApplications = tenantsCopy().values().stream() + .mapToInt(tenant -> tenant.getApplicationRepo().listApplications().size()).sum(); + int applicationsRedeployed = 0; + for (Tenant tenant : tenantsCopy().values()) { + tenant.redeployApplications(deployer); + applicationsRedeployed += redeployProgress(tenant, applicationsRedeployed, totalNumberOfApplications); + } + } + + private static int redeployProgress(Tenant tenant, int applicationsRedeployed, int totalNumberOfApplications) { + int size = tenant.getApplicationRepo().listApplications().size(); + if (size > 0) { + log.log(LogLevel.INFO, String.format("Redeployed %s of %s applications", applicationsRedeployed + size, totalNumberOfApplications)); + } + return size; + } + + /** + * Gets zookeeper path for tenant data + * @param tenantName tenant name + * @return a {@link com.yahoo.path.Path} to the zookeeper data for a tenant + */ + public static Path getTenantPath(TenantName tenantName) { + return tenantsPath.append(tenantName.value()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java new file mode 100644 index 00000000000..fe02cc18bd8 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/TimeoutBudget.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.List; + +/** + * Handles a timeout logic by providing higher level abstraction for asking if there is time left. + * + * @author lulf + * @since 5.1 + */ +public class TimeoutBudget { + private final Clock clock; + private final Instant startTime; + private final List<Instant> measurements = new ArrayList<>(); + private final Instant endTime; + + public TimeoutBudget(Clock clock, Duration duration) { + this.clock = clock; + this.startTime = clock.instant(); + this.endTime = startTime.plus(duration); + } + + public Duration timeLeft() { + Instant now = clock.instant(); + measurements.add(now); + Duration duration = Duration.between(now, endTime); + return duration.isNegative() ? Duration.ofMillis(0) : duration; + } + + public boolean hasTimeLeft() { + Instant now = clock.instant(); + measurements.add(now); + return now.isBefore(endTime); + } + + public String timesUsed() { + StringBuilder buf = new StringBuilder(); + buf.append("["); + Instant prev = startTime; + for (Instant m : measurements) { + buf.append(Duration.between(prev, m).toMillis()); + prev = m; + buf.append(" ms, "); + } + Instant now = clock.instant(); + buf.append("total: "); + buf.append(Duration.between(startTime, now).toMillis()); + buf.append(" ms]"); + return buf.toString(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UncompressedConfigResponseFactory.java new file mode 100644 index 00000000000..73e7acb5a88 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UncompressedConfigResponseFactory.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.Utf8Array; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.protocol.CompressionInfo; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.SlimeConfigResponse; +import com.yahoo.vespa.config.util.ConfigUtils; + +/** + * Simply returns an uncompressed payload. + * + * @author lulf + * @since 5.19 + */ +public class UncompressedConfigResponseFactory implements ConfigResponseFactory { + @Override + public ConfigResponse createResponse(ConfigPayload payload, InnerCNode defFile, long generation) { + Utf8Array rawPayload = payload.toUtf8Array(true); + String configMd5 = ConfigUtils.getMd5(rawPayload); + CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength()); + return new SlimeConfigResponse(rawPayload, defFile, generation, configMd5, info); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UnknownConfigDefinitionException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UnknownConfigDefinitionException.java new file mode 100644 index 00000000000..182c25a84e2 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UnknownConfigDefinitionException.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +/** + * Indicates that a config definition (typically a def file schema) was unknown to the config server + * + * @author lulf + * @since 5.1 + */ +public class UnknownConfigDefinitionException extends IllegalArgumentException { + public UnknownConfigDefinitionException(String s) { + super(s); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/VersionDoesNotExistException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/VersionDoesNotExistException.java new file mode 100644 index 00000000000..d6a329491e6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/VersionDoesNotExistException.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +/** + * @author Vegard Sjonfjell + */ +public final class VersionDoesNotExistException extends RuntimeException { + public VersionDoesNotExistException(String message) { + super(message); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java new file mode 100644 index 00000000000..7116dda7322 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -0,0 +1,243 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigCacheKey; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.server.ConfigResponseFactory; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.UncompressedConfigResponseFactory; +import com.yahoo.vespa.config.server.UnknownConfigDefinitionException; +import com.yahoo.vespa.config.server.modelfactory.ModelResult; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +/** + * A Vespa application for a specific version of Vespa. It holds data and metadata associated with + * a Vespa application, i.e. generation, vespamodel instance and zookeeper data, as well as methods for resolving config + * and other queries against the model. + * + * @author Harald Musum + * @since 2010-12-08 + */ +public class Application implements ModelResult { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Application.class.getName()); + private final long appGeneration; // The generation of the set of configs belonging to an application + private final Version vespaVersion; + private final Model model; + private final ServerCache cache; + private final MetricUpdater metricUpdater; + private final ApplicationId app; + + public Application(Model model, ServerCache cache, long appGeneration, Version vespaVersion, MetricUpdater metricUpdater, ApplicationId app) { + Objects.requireNonNull(model, "The model cannot be null"); + this.model = model; + this.cache = cache; + this.appGeneration = appGeneration; + this.vespaVersion = vespaVersion; + this.metricUpdater = metricUpdater; + this.app = app; + } + + /** + * Returns the generation for the config we are currently serving + * + * @return the config generation + */ + public Long getApplicationGeneration() { return appGeneration; } + + // TODO: Return ApplicationName + public String getName() { return app.application().value(); } + + /** Returns the application model, never null */ + @Override + public Model getModel() { return model; } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("application '").append(app.application().value()).append("', "); + sb.append("generation ").append(appGeneration).append(", "); + sb.append("vespa version ").append(vespaVersion); + return sb.toString(); + } + + public ServerCache getCache() { + return cache; + } + + public ApplicationId getId() { + return app; + } + + public Version getVespaVersion() { + return vespaVersion; + } + + /** + * The old style (deprecated) configs/ user overrides for this key + * + * @param key the key for the config to get user config for + * @return the user config value or null + */ + private ConfigPayload getLegacyUserConfigs(ConfigCacheKey key) { + try { + if (logDebug()) { + debug("Looking up legacy user config for " + key); + } + return cache.getLegacyUserConfig(key.getKey().getName()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Gets a config from ZK. Returns null if not found. + */ + public ConfigResponse resolveConfig(GetConfigRequest req, ConfigResponseFactory responseFactory) { + long start = System.currentTimeMillis(); + metricUpdater.incrementRequests(); + ConfigKey<?> configKey = req.getConfigKey(); + String defMd5 = configKey.getMd5(); + if (defMd5 == null || defMd5.isEmpty()) { + defMd5 = ConfigUtils.getDefMd5(req.getDefContent().asList()); + } + ConfigCacheKey cacheKey = new ConfigCacheKey(configKey, defMd5); + if (logDebug()) { + debug("Resolving config " + cacheKey); + } + + if (!req.noCache()) { + ConfigResponse config = cache.get(cacheKey); + if (config != null) { + if (logDebug()) { + debug("Found config " + cacheKey + " in cache"); + } + metricUpdater.incrementProcTime(System.currentTimeMillis() - start); + return config; + } + } + + // Try new ConfigInstance based API: + ConfigDefinitionWrapper configDefinitionWrapper = getTargetDef(req); + ConfigDefinition def = configDefinitionWrapper.getDef(); + if (def == null) { + metricUpdater.incrementFailedRequests(); + throw new UnknownConfigDefinitionException("Unable to find config definition for '" + configKey.getNamespace() + "." + configKey.getName()); + } + configKey = new ConfigKey<>(configDefinitionWrapper.getDefKey().getName(), configKey.getConfigId(), configDefinitionWrapper.getDefKey().getNamespace()); + ConfigPayload override = getLegacyUserConfigs(cacheKey); + ConfigPayload payload; + try { + if (logDebug()) { + debug("Resolving " + configKey + " with targetDef=" + def + ", override=" + override); + } + + payload = model.getConfig( + configKey, + def, + override); + } catch (IOException e) { + metricUpdater.incrementFailedRequests(); + throw new ConfigurationRuntimeException("Unable to resolve config", e); + } + + ConfigResponse configResponse = responseFactory.createResponse(payload, def.getCNode(), appGeneration); + metricUpdater.incrementProcTime(System.currentTimeMillis() - start); + if (!req.noCache()) { + cache.put(cacheKey, configResponse, configResponse.getConfigMd5()); + metricUpdater.setCacheConfigElems(cache.configElems()); + metricUpdater.setCacheChecksumElems(cache.checkSumElems()); + } + return configResponse; + } + + private boolean logDebug() { + return log.isLoggable(LogLevel.DEBUG); + } + + private void debug(String message) { + log.log(LogLevel.DEBUG, Tenants.logPre(getId())+message); + } + + private ConfigDefinitionWrapper getTargetDef(GetConfigRequest req) { + ConfigKey<?> configKey = req.getConfigKey(); + DefContent def = req.getDefContent(); + ConfigDefinitionKey configDefinitionKey = new ConfigDefinitionKey(configKey.getName(), configKey.getNamespace()); + if (def.isEmpty()) { + if (logDebug()) { + debug("No config schema in request for " + configKey); + } + ConfigDefinition ret = cache.getDef(configDefinitionKey); + return new ConfigDefinitionWrapper(configDefinitionKey, ret); + } else { + if (logDebug()) { + debug("Got config schema from request, length:" + def.asList().size() + " : " + configKey); + } + return new ConfigDefinitionWrapper(configDefinitionKey, new ConfigDefinition(configKey.getName(), def.asStringArray())); + } + } + + public void updateHostMetrics(int numHosts) { + metricUpdater.setHosts(numHosts); + } + + // For testing only + ConfigResponse resolveConfig(GetConfigRequest req) { + return resolveConfig(req, new UncompressedConfigResponseFactory()); + } + + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, String configId) throws IOException { + ConfigKey<CONFIGTYPE> key = new ConfigKey<>(configClass, configId); + ConfigPayload payload = model.getConfig(key, (ConfigDefinition)null, null); + return payload.toInstance(configClass, configId); + } + + /** + * Wrapper class for holding config definition key and def, since when looking up + * we may end up changing the config definition key (fallback mechanism when using + * legacy config namespace (or not using config namespace)) + */ + private static class ConfigDefinitionWrapper { + private final ConfigDefinitionKey defKey; + private final ConfigDefinition def; + + ConfigDefinitionWrapper(ConfigDefinitionKey defKey, ConfigDefinition def) { + this.defKey = defKey; + this.def = def; + } + + public ConfigDefinitionKey getDefKey() { + return defKey; + } + + public ConfigDefinition getDef() { + return def; + } + } + + public Set<ConfigKey<?>> allConfigsProduced() { + return model.allConfigsProduced(); + } + + public Set<String> allConfigIds() { + return model.allConfigIds(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java new file mode 100644 index 00000000000..a36167517d8 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceChecker.java @@ -0,0 +1,262 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import org.glassfish.jersey.client.proxy.WebResourceFactory; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Checks for convergence of config generation for a given application. + * + * @author lulf + */ +public class ApplicationConvergenceChecker extends AbstractComponent { + private static final String statePath = "/state/v1/"; + private static final String configSubPath = "config"; + private static final String configPath = statePath + configSubPath; + private final StateApiFactory stateApiFactory; + private final Client client = ClientBuilder.newClient(); + + private final static Set<String> serviceTypes = new HashSet<>(Arrays.asList( + "container", + "qrserver", + "docprocservice", + "searchnode", + "storagenode", + "distributor" + )); + + @Inject + public ApplicationConvergenceChecker() { + this(ApplicationConvergenceChecker::createStateApi); + } + + public ApplicationConvergenceChecker(StateApiFactory stateApiFactory) { + this.stateApiFactory = stateApiFactory; + } + + // TODO: Remove this function once the other has taken over (list) + private void waitForConfigConverged(ModelConfig config, long wantedGeneration, TimeoutBudget timeoutBudget) { + + config.hosts().stream() + .forEach(host -> host.services().stream() + .filter(service -> serviceTypes.contains(service.type())) + .forEach(service -> { + Optional<Integer> statePort = getStatePort(service); + if (statePort.isPresent()) { + URI serviceUri = getServiceUri(host.name(), statePort.get()); + try { + waitForServiceGenerationConverged(serviceUri, wantedGeneration, timeoutBudget); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + })); + } + + public void waitForConfigConverged(Application application, TimeoutBudget timeoutBudget) throws IOException { + ModelConfig config = application.getConfig(ModelConfig.class, ""); + waitForConfigConverged(config, application.getApplicationGeneration(), timeoutBudget); + } + + private Optional<Integer> getStatePort(ModelConfig.Hosts.Services service) { return service.ports().stream() + .filter(port -> port.tags().contains("state")) + .map(ModelConfig.Hosts.Services.Ports::number) + .findFirst(); + } + + private URI getServiceUri(String host, int port) { + return URI.create("http://" + host + ":" + port); + } + + private void waitForServiceGenerationConverged(URI serviceUri, long wantedGeneration, TimeoutBudget timeoutBudget) throws InterruptedException { + long generation = -1; + do { + try { + generation = getServiceGeneration(serviceUri); + if (generation >= wantedGeneration) + return; + } catch (Exception e) { + // Try again + } + Thread.sleep(100); + } while (timeoutBudget.hasTimeLeft()); + StringBuilder message = new StringBuilder("Timed out waiting for service to use config generation "). + append(wantedGeneration). + append(" (checking "). + append(serviceUri).append(configPath). + append("), "); + if (generation == -1) { + message.append("could not connect."); + } else { + message.append("generation was ").append(generation).append("."); + } + throw new ConfigNotConvergedException(message.toString()); + } + + public long generationFromContainerState(JsonNode state) { + return state.get("config").get("generation").asLong(); + } + + private static StateApi createStateApi(Client client, URI uri) { + WebTarget target = client.target(uri); + return WebResourceFactory.newResource(StateApi.class, target); + } + + private long getServiceGeneration(URI serviceUri) { + StateApi state = stateApiFactory.createStateApi(client, serviceUri); + return generationFromContainerState(state.config()); + } + + @Override + public void deconstruct() { + client.close(); + } + + private boolean hostInApplication(Application application, String hostPort) { + final ModelConfig config; + try { + config = application.getConfig(ModelConfig.class, ""); + } catch (IOException e) { + throw new RuntimeException(e); + } + final List<ModelConfig.Hosts> hosts = config.hosts(); + for (ModelConfig.Hosts host : hosts) { + if (hostPort.startsWith(host.name())) { + for (ModelConfig.Hosts.Services service : host.services()) { + for (ModelConfig.Hosts.Services.Ports port : service.ports()) { + if (hostPort.equals(host.name() + ":" + port.number())) { + return true; + } + } + } + } + } + return false; + } + + public HttpResponse nodeConvergenceCheck(Application application, String hostFromRequest, URI uri) { + JSONObject answer = new JSONObject(); + JSONObject debug = new JSONObject(); + try { + answer.put("url", uri); + debug.put("wantedGeneration", application.getApplicationGeneration()); + debug.put("host", hostFromRequest); + + if (!hostInApplication(application, hostFromRequest)) { + debug.put("problem", "Host:port (service) no longer part of application, refetch list of services."); + answer.put("debug", debug); + return new JsonHttpResponse(410, answer); + } + final long generation = getServiceGeneration(URI.create("http://" + hostFromRequest)); + debug.put("currentGeneration", generation); + answer.put("debug", debug); + answer.put("converged", generation >= application.getApplicationGeneration()); + return new JsonHttpResponse(200, answer); + } catch(JSONException e) { + try { + answer.put("error", e.getMessage()); + } catch (JSONException e1) { + throw new RuntimeException("Fail while creating error message ", e1); + } + return new JsonHttpResponse(500, answer); + } + } + + private static class JsonHttpResponse extends HttpResponse { + + private final JSONObject answer; + + JsonHttpResponse(int returncode, JSONObject answer) { + super(returncode); + this.answer = answer; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(answer.toString().getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + } + + public HttpResponse listConfigConvergence(Application application, URI uri) { + final JSONObject answer = new JSONObject(); + final JSONArray nodes = new JSONArray(); + final ModelConfig config; + try { + config = application.getConfig(ModelConfig.class, ""); + } catch (IOException e) { + throw new RuntimeException("failed on get config model", e); + } + config.hosts().stream() + .forEach(host -> { + host.services().stream() + .filter(service -> serviceTypes.contains(service.type())) + .forEach(service -> { + Optional<Integer> statePort = getStatePort(service); + if (statePort.isPresent()) { + JSONObject hostNode = new JSONObject(); + try { + hostNode.put("host", host.name()); + hostNode.put("port", statePort.get()); + hostNode.put("url", uri.toString() + "/" + host.name() + ":" + statePort.get()); + hostNode.put("type", service.type()); + + } catch (JSONException e) { + throw new RuntimeException(e); + } + nodes.put(hostNode); + } + }); + }); + try { + answer.put("services", nodes); + JSONObject debug = new JSONObject(); + debug.put("wantedVersion", application.getApplicationGeneration()); + answer.put("debug", debug); + answer.put("url", uri.toString()); + return new JsonHttpResponse(200, answer); + } catch (JSONException e) { + try { + answer.put("error", e.getMessage()); + } catch (JSONException e1) { + throw new RuntimeException("Failed while creating error message ", e1); + } + return new JsonHttpResponse(500, answer); + } + } + + @Path(statePath) + public interface StateApi { + @Path(configSubPath) + @GET + JsonNode config(); + } + + public interface StateApiFactory { + StateApi createStateApi(Client client, URI serviceUri); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationRepo.java new file mode 100644 index 00000000000..a9296f26eeb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationRepo.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Transaction; + +import java.util.List; + +/** + * Manages the list of active applications in a config server. + * + * @author lulf + * @since 5.1 + */ +public interface ApplicationRepo { + + /** + * List the active applications in this config server. + * + * @return a list of {@link com.yahoo.config.provision.ApplicationId}s that are active. + */ + public List<ApplicationId> listApplications(); + + /** + * Register active application and adds it to the repo. If it already exists it is overwritten. + * + * @param applicationId An {@link com.yahoo.config.provision.ApplicationId} that represents an active application. + * @param sessionId Id of the session containing the application package for this id. + */ + Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId); + + /** + * Return the stored session id for a given application. + * + * @param applicationId an {@link ApplicationId} + * @return session id of given application id. + * @throws IllegalArgumentException if the application does not exist + */ + long getSessionIdForApplication(ApplicationId applicationId); + + /** + * Deletes an application from the repo if it exists. + * + * @param applicationId an {@link ApplicationId} to delete. + */ + void deleteApplication(ApplicationId applicationId); + + /** + * Closes the application repo. Once a repo has been closed, it should not be used again. + */ + void close(); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java new file mode 100644 index 00000000000..91e57d504a5 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +/** + * @author lulf + */ +public class ConfigNotConvergedException extends RuntimeException { + public ConfigNotConvergedException(String message) { + super(message); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java new file mode 100644 index 00000000000..bd60527ca79 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.component.AbstractComponent; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.yolean.Exceptions; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * Fetches log entries from logserver with level errors and fatal. The logserver only return + * a log entry once over this API so doing repeated call will not give the same results. + * + * @author dybdahl + */ +public class LogServerLogGrabber extends AbstractComponent { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(LogServerLogGrabber.class.getName()); + + @Inject + public LogServerLogGrabber() {} + + private Optional<Integer> getErrorLogPort(ModelConfig.Hosts.Services service) { + return service.ports().stream() + .filter(port -> port.tags().toLowerCase().contains("last-errors-holder")) + .map(ModelConfig.Hosts.Services.Ports::number) + .findFirst(); + } + + private class LogServerConnectionInfo { + String hostName; + int port; + } + + public HttpResponse grabLog(Application application) { + + final ModelConfig config; + try { + config = application.getConfig(ModelConfig.class, ""); + } catch (IOException e) { + throw new RuntimeException(e); + } + + final LogServerConnectionInfo logServerConnectionInfo = new LogServerConnectionInfo(); + + config.hosts().stream() + .forEach(host -> host.services().stream() + .filter(service -> service.type().equals("logserver")) + .forEach(logService -> { + Optional<Integer> logPort = getErrorLogPort(logService); + if (logPort.isPresent()) { + if (logServerConnectionInfo.hostName != null) { + throw new RuntimeException("Found several log server ports."); + } + logServerConnectionInfo.hostName = host.name(); + logServerConnectionInfo.port = logPort.get(); + } + })); + + if (logServerConnectionInfo.hostName == null) { + return new HttpResponse(503) { + @Override + public void render(OutputStream outputStream) throws IOException { + PrintWriter printWriter = new PrintWriter(outputStream); + printWriter.print("Did not find any log server in config model."); + printWriter.close(); + } + }; + } + log.log(LogLevel.DEBUG, "Requested error logs, pulling from logserver on " + logServerConnectionInfo.hostName + " " + + logServerConnectionInfo.port); + final String response; + try { + response = readLog(logServerConnectionInfo.hostName, logServerConnectionInfo.port); + log.log(LogLevel.DEBUG, "Requested error logs was " + response.length() + " characters"); + } catch (IOException e) { + return HttpErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(response.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + }; + } + + private String readLog(String host, int port) throws IOException { + Socket socket = new Socket(host, port); + BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream())); + StringBuilder data = new StringBuilder(); + + int bufferSize = 4096; + int charsRead; + do { + char[] buffer = new char[bufferSize]; + charsRead = in.read(buffer); + data.append(new String(buffer, 0, charsRead)); + } while (charsRead == bufferSize); + in.close(); + socket.close(); + return data.toString(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java new file mode 100644 index 00000000000..abffb002a76 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.log.LogLevel; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * A global permanent application package containing configuration info that is always used during deploy. + * + * @author lulf + * @since 5.15 + */ +public class PermanentApplicationPackage { + + private static final Logger log = Logger.getLogger(PermanentApplicationPackage.class.getName()); + private final Optional<ApplicationPackage> applicationPackage; + + public PermanentApplicationPackage(ConfigserverConfig config) { + File app = new File(Defaults.getDefaults().underVespaHome(config.applicationDirectory())); + applicationPackage = Optional.<ApplicationPackage>ofNullable(app.exists() ? + FilesApplicationPackage.fromFile(app) : null); + if (applicationPackage.isPresent()) { + log.log(LogLevel.DEBUG, "Detected permanent application package in '" + + Defaults.getDefaults().underVespaHome(config.applicationDirectory()) + + "'. This might add extra services to config models"); + } + } + + /** + * Get the permanent application package. + * + * @return An {@link Optional} of the application package, as it may not exist. + */ + public Optional<ApplicationPackage> applicationPackage() { + return applicationPackage; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKApplicationRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKApplicationRepo.java new file mode 100644 index 00000000000..6567f8439a5 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKApplicationRepo.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +/** + * Application repo backed by zookeeper. + * + * @author lulf + * @since 5.1 + */ +public class ZKApplicationRepo implements ApplicationRepo, PathChildrenCacheListener { + + private static final Logger log = Logger.getLogger(ZKApplicationRepo.class.getName()); + private final Curator curator; + private final Path root; + private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKApplicationRepo.class.getName())); + private final Curator.DirectoryCache directoryCache; + private final ReloadHandler reloadHandler; + private final TenantName tenant; + + private ZKApplicationRepo(Curator curator, Path root, ReloadHandler reloadHandler, TenantName tenant) throws Exception { + this.curator = curator; + this.root = root; + this.reloadHandler = reloadHandler; + this.tenant = tenant; + rewriteApplicationIds(); + this.directoryCache = curator.createDirectoryCache(root.getAbsolute(), false, false, pathChildrenExecutor); + this.directoryCache.start(); + this.directoryCache.addListener(this); + } + + private void rewriteApplicationIds() { + try { + List<String> appNodes = curator.framework().getChildren().forPath(root.getAbsolute()); + for (String appNode : appNodes) { + Optional<ApplicationId> appId = parseApplication(appNode); + appId.filter(id -> shouldBeRewritten(appNode, id)) + .ifPresent(id -> rewriteApplicationId(id, appNode, readSessionId(id, appNode))); + } + } catch (Exception e) { + log.log(LogLevel.WARNING, "Error rewriting application ids on upgrade", e); + } + } + + private long readSessionId(ApplicationId appId, String appNode) { + String path = root.append(appNode).getAbsolute(); + try { + return Long.parseLong(Utf8.toString(curator.framework().getData().forPath(path))); + } catch (Exception e) { + throw new IllegalArgumentException(Tenants.logPre(appId) + "Unable to read the session id from '" + path + "'", e); + } + } + + private boolean shouldBeRewritten(String appNode, ApplicationId appId) { + return !appNode.equals(appId.serializedForm()); + } + + private void rewriteApplicationId(ApplicationId appId, String origNode, long sessionId) { + String newPath = root.append(appId.serializedForm()).getAbsolute(); + String oldPath = root.append(origNode).getAbsolute(); + try (CuratorTransaction transaction = new CuratorTransaction(curator)) { + if (curator.framework().checkExists().forPath(newPath) == null) { + transaction.add(CuratorOperations.create(newPath, Utf8.toAsciiBytes(sessionId))); + } + transaction.add(CuratorOperations.delete(oldPath)); + transaction.commit(); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Error rewriting application id from " + origNode + " to " + appId.serializedForm()); + } + } + + public static ApplicationRepo create(Curator curator, Path root, ReloadHandler reloadHandler, TenantName tenant) { + try { + return new ZKApplicationRepo(curator, root, reloadHandler, tenant); + } catch (Exception e) { + throw new RuntimeException(Tenants.logPre(tenant)+"Error creating application repo", e); + } + } + + @Override + public List<ApplicationId> listApplications() { + try { + List<String> appNodes = curator.framework().getChildren().forPath(root.getAbsolute()); + List<ApplicationId> applicationIds = new ArrayList<>(); + for (String appNode : appNodes) { + parseApplication(appNode).ifPresent(applicationIds::add); + } + return applicationIds; + } catch (Exception e) { + throw new RuntimeException(Tenants.logPre(tenant)+"Unable to list applications", e); + } + } + + private Optional<ApplicationId> parseApplication(String appNode) { + try { + return Optional.of(ApplicationId.fromSerializedForm(tenant, appNode)); + } catch (IllegalArgumentException e) { + log.log(LogLevel.INFO, Tenants.logPre(tenant)+"Unable to parse application with id '" + appNode + "', ignoring."); + return Optional.empty(); + } + } + + @Override + public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { + if (listApplications().contains(applicationId)) { + return new CuratorTransaction(curator).add(CuratorOperations.setData(root.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + } else { + return new CuratorTransaction(curator).add(CuratorOperations.create(root.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + } + } + + @Override + public long getSessionIdForApplication(ApplicationId applicationId) { + return readSessionId(applicationId, applicationId.serializedForm()); + } + + @Override + public void deleteApplication(ApplicationId applicationId) { + Path path = root.append(applicationId.serializedForm()); + curator.delete(path); + } + + @Override + public void close() { + directoryCache.close(); + pathChildrenExecutor.shutdown(); + } + + + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + switch (event.getType()) { + case CHILD_ADDED: + applicationAdded(ApplicationId.fromSerializedForm(tenant, Path.fromString(event.getData().getPath()).getName())); + break; + case CHILD_REMOVED: + applicationRemoved(ApplicationId.fromSerializedForm(tenant, Path.fromString(event.getData().getPath()).getName())); + break; + } + } + + private void applicationRemoved(ApplicationId applicationId) { + reloadHandler.removeApplication(applicationId); + log.log(LogLevel.DEBUG, Tenants.logPre(applicationId)+"Application removed: " + applicationId); + } + + private void applicationAdded(ApplicationId applicationId) { + log.log(LogLevel.DEBUG, Tenants.logPre(applicationId)+"Application " + applicationId + " was added to repo"); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java new file mode 100644 index 00000000000..7c91a57c3af --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ConfigChangeAction; + +import java.util.List; + +/** + * Contains an aggregated view of which actions that must be performed to handle config + * changes between the current active model and the next model to prepare. + * The actions are split into restart and re-feed actions. + * + * @author geirst + * @since 5.44 + */ +public class ConfigChangeActions { + + private final RestartActions restartActions; + private final RefeedActions refeedActions; + + public ConfigChangeActions() { + this.restartActions = new RestartActions(); + this.refeedActions = new RefeedActions(); + } + + public ConfigChangeActions(List<ConfigChangeAction> actions) { + this.restartActions = new RestartActions(actions); + this.refeedActions = new RefeedActions(actions); + } + + public RestartActions getRestartActions() { + return restartActions; + } + + public RefeedActions getRefeedActions() { + return refeedActions; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java new file mode 100644 index 00000000000..5a426182b1a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverter.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.slime.Cursor; + +import java.util.Set; + +/** + * Class used to convert a ConfigChangeActions instance to Slime. + * + * @author geirst + * @since 5.44 + */ +public class ConfigChangeActionsSlimeConverter { + private final ConfigChangeActions actions; + + public ConfigChangeActionsSlimeConverter(ConfigChangeActions actions) { + this.actions = actions; + } + + public void toSlime(Cursor root) { + Cursor actionsCursor = root.setObject("configChangeActions"); + restartActionsToSlime(actionsCursor); + refeedActionsToSlime(actionsCursor); + } + + private void restartActionsToSlime(Cursor actionsCursor) { + Cursor restartCursor = actionsCursor.setArray("restart"); + for (RestartActions.Entry entry : actions.getRestartActions().getEntries()) { + Cursor entryCursor = restartCursor.addObject(); + entryCursor.setString("clusterName", entry.getClusterName()); + entryCursor.setString("clusterType", entry.getClusterType()); + entryCursor.setString("serviceType", entry.getServiceType()); + messagesToSlime(entryCursor, entry.getMessages()); + servicesToSlime(entryCursor, entry.getServices()); + } + } + + private void refeedActionsToSlime(Cursor actionsCursor) { + Cursor refeedCursor = actionsCursor.setArray("refeed"); + for (RefeedActions.Entry entry : actions.getRefeedActions().getEntries()) { + Cursor entryCursor = refeedCursor.addObject(); + entryCursor.setString("name", entry.name()); + entryCursor.setBool("allowed", entry.allowed()); + entryCursor.setString("documentType", entry.getDocumentType()); + entryCursor.setString("clusterName", entry.getClusterName()); + messagesToSlime(entryCursor, entry.getMessages()); + servicesToSlime(entryCursor, entry.getServices()); + } + } + + private static void messagesToSlime(Cursor entryCursor, Set<String> messages) { + Cursor messagesCursor = entryCursor.setArray("messages"); + for (String message : messages) { + messagesCursor.addString(message); + } + } + + private static void servicesToSlime(Cursor entryCursor, Set<ServiceInfo> services) { + Cursor servicesCursor = entryCursor.setArray("services"); + for (ServiceInfo service : services) { + Cursor serviceCursor = servicesCursor.addObject(); + serviceCursor.setString("serviceName", service.getServiceName()); + serviceCursor.setString("serviceType", service.getServiceType()); + serviceCursor.setString("configId", service.getConfigId()); + serviceCursor.setString("hostName", service.getHostName()); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java new file mode 100644 index 00000000000..87b7b466fbe --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActions.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ConfigChangeRefeedAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.*; + +/** + * Represents all actions to re-feed document types in order to handle config changes. + * + * @author geirst + * @since 5.44 + */ +public class RefeedActions { + + public static class Entry { + + private final String name; + private final boolean allowed; + private final String documentType; + private final String clusterName; + private final Set<ServiceInfo> services = new LinkedHashSet<>(); + private final Set<String> messages = new TreeSet<>(); + + private Entry(String name, boolean allowed, String documentType, String clusterName) { + this.name = name; + this.allowed = allowed; + this.documentType = documentType; + this.clusterName = clusterName; + } + + private Entry addService(ServiceInfo service) { + services.add(service); + return this; + } + + private Entry addMessage(String message) { + messages.add(message); + return this; + } + + public String name() { return name; } + + public boolean allowed() { return allowed; } + + public String getDocumentType() { return documentType; } + + public String getClusterName() { return clusterName; } + + public Set<ServiceInfo> getServices() { return services; } + + public Set<String> getMessages() { return messages; } + + } + + private Entry addEntry(String name, boolean allowed, String documentType, ServiceInfo service) { + String clusterName = service.getProperty("clustername").orElse(""); + String entryId = name + "." + allowed + "." + clusterName + "." + documentType; + Entry entry = actions.get(entryId); + if (entry == null) { + entry = new Entry(name, allowed, documentType, clusterName); + actions.put(entryId, entry); + } + return entry; + } + + private final Map<String, Entry> actions = new TreeMap<>(); + + public RefeedActions() { + } + + public RefeedActions(List<ConfigChangeAction> actions) { + for (ConfigChangeAction action : actions) { + if (action.getType().equals(ConfigChangeAction.Type.REFEED)) { + ConfigChangeRefeedAction refeedAction = (ConfigChangeRefeedAction) action; + for (ServiceInfo service : refeedAction.getServices()) { + addEntry(refeedAction.name(), refeedAction.allowed(), refeedAction.getDocumentType(), service). + addService(service). + addMessage(action.getMessage()); + } + } + } + } + + public List<Entry> getEntries() { + return new ArrayList<>(actions.values()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java new file mode 100644 index 00000000000..ae72c61bcdb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatter.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +/** + * Class used to format re-feed actions for human readability. + * + * @author geirst + * @since 5.44 + */ +public class RefeedActionsFormatter { + + private final RefeedActions actions; + + public RefeedActionsFormatter(RefeedActions actions) { + this.actions = actions; + } + + public String format() { + StringBuilder builder = new StringBuilder(); + for (RefeedActions.Entry entry : actions.getEntries()) { + if (entry.allowed()) + builder.append("(allowed) "); + builder.append(entry.name() + ": Consider removing data and re-feed document type '" + entry.getDocumentType() + + "' in cluster '" + entry.getClusterName() + "' because:\n"); + int counter = 1; + for (String message : entry.getMessages()) { + builder.append(" " + (counter++) + ") " + message + "\n"); + } + } + return builder.toString(); + } + +}
\ No newline at end of file diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java new file mode 100644 index 00000000000..6c2c080e6e4 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.*; + +/** + * Represents all actions to restart services in order to handle a config change. + * + * @author geirst + * @since 5.44 + */ +public class RestartActions { + + public static class Entry { + + private final String clusterName; + private final String clusterType; + private final String serviceType; + private final Set<ServiceInfo> services = new LinkedHashSet<>(); + private final Set<String> messages = new TreeSet<>(); + + private Entry addService(ServiceInfo service) { + services.add(service); + return this; + } + + private Entry addMessage(String message) { + messages.add(message); + return this; + } + + private Entry(String clusterName, String clusterType, String serviceType) { + this.clusterName = clusterName; + this.clusterType = clusterType; + this.serviceType = serviceType; + } + + public String getClusterName() { + return clusterName; + } + + public String getClusterType() { + return clusterType; + } + + public String getServiceType() { + return serviceType; + } + + public Set<ServiceInfo> getServices() { + return services; + } + + public Set<String> getMessages() { + return messages; + } + + } + + private Entry addEntry(ServiceInfo service) { + String clusterName = service.getProperty("clustername").orElse(""); + String clusterType = service.getProperty("clustertype").orElse(""); + String entryId = clusterType + "." + clusterName + "." + service.getServiceType(); + Entry entry = actions.get(entryId); + if (entry == null) { + entry = new Entry(clusterName, clusterType, service.getServiceType()); + actions.put(entryId, entry); + } + return entry; + } + + private final Map<String, Entry> actions = new TreeMap<>(); + + public RestartActions() { + } + + public RestartActions(List<ConfigChangeAction> actions) { + for (ConfigChangeAction action : actions) { + if (action.getType().equals(ConfigChangeAction.Type.RESTART)) { + for (ServiceInfo service : action.getServices()) { + addEntry(service). + addService(service). + addMessage(action.getMessage()); + } + } + } + } + + public List<Entry> getEntries() { + return new ArrayList<>(actions.values()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java new file mode 100644 index 00000000000..7d61f31ac47 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +/** + * Class used to format restart actions for human readability. + * + * @author geirst + * @since 5.44 + */ +public class RestartActionsFormatter { + + private final RestartActions actions; + + public RestartActionsFormatter(RestartActions actions) { + this.actions = actions; + } + + public String format() { + StringBuilder builder = new StringBuilder(); + for (RestartActions.Entry entry : actions.getEntries()) { + builder.append("In cluster '" + entry.getClusterName() + "' of type '" + entry.getClusterType() + "':\n"); + builder.append(" Restart services of type '" + entry.getServiceType() + "' because:\n"); + int counter = 1; + for (String message : entry.getMessages()) { + builder.append(" " + counter++ + ") " + message + "\n"); + } + } + return builder.toString(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/counter/package-info.java b/configserver/src/main/java/com/yahoo/vespa/config/server/counter/package-info.java new file mode 100644 index 00000000000..219e1008f3a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/counter/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author andreer + */ +@ExportPackage +package com.yahoo.vespa.config.server.counter; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployer.java new file mode 100644 index 00000000000..00a036df9d5 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployer.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.vespa.config.server.ActivateLock; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.curator.Curator; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; + +/** + * The API for deploying applications. + * A class which needs to deploy applications can have an instance of this injected. + * + * @author bratseth + */ +public class Deployer implements com.yahoo.config.provision.Deployer { + + private final Tenants tenants; + private final Optional<Provisioner> hostProvisioner; + private final ConfigserverConfig configserverConfig; + private final Curator curator; + private final Clock clock; + private final DeployLogger logger = new SilentDeployLogger(); + + public Deployer(Tenants tenants, HostProvisionerProvider hostProvisionerProvider, + ConfigserverConfig configserverConfig, Curator curator) { + this.tenants = tenants; + this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); + this.configserverConfig = configserverConfig; + this.curator = curator; + this.clock = Clock.systemUTC(); + } + + /** + * Creates a new deployment from the active application, if available. + * + * @param application the active application to be redeployed + * @param timeout the timeout to use for each individual deployment operation + * @return a new deployment from the local active, or empty if a local active application + * was not present for this id (meaning it either is not active or active on another + * node in the config server cluster) + */ + @Override + public Optional<com.yahoo.config.provision.Deployment> deployFromLocalActive(ApplicationId application, Duration timeout) { + Tenant tenant = tenants.tenantsCopy().get(application.tenant()); + LocalSession activeSession = tenant.getLocalSessionRepo().getActiveSession(application); + if (activeSession == null) return Optional.empty(); + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, timeoutBudget); + tenant.getLocalSessionRepo().addSession(newSession); + return Optional.of(Deployment.unprepared(newSession, + tenant.getLocalSessionRepo(), + tenant.getPath(), + configserverConfig, + hostProvisioner, + new ActivateLock(curator, tenant.getPath()), + timeout, clock)); + } + + public Deployment deployFromPreparedSession(LocalSession session, ActivateLock lock, LocalSessionRepo localSessionRepo, Duration timeout) { + return Deployment.prepared(session, + localSessionRepo, + hostProvisioner, + lock, + timeout, clock); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java new file mode 100644 index 00000000000..cf366fd21f7 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.ActivateLock; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.session.Session; +import com.yahoo.vespa.config.server.session.SilentDeployLogger; + +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * The process of deploying an application. + * Deployments are created by a {@link Deployer}. + * Instances of this are not multithread safe. + * + * @author lulf + * @author bratseth + */ +public class Deployment implements com.yahoo.config.provision.Deployment { + + private static final Logger log = Logger.getLogger(Deployment.class.getName()); + + /** The session containing the application instance to activate */ + private final LocalSession session; + private final LocalSessionRepo localSessionRepo; + /** The path to the tenant, or null if not available (only used during prepare) */ + private final Path tenantPath; + /** The config server config, or null if not available (only used during prepare) */ + private final ConfigserverConfig configserverConfig; + private final Optional<Provisioner> hostProvisioner; + private final ActivateLock activateLock; + private final Duration timeout; + private final Clock clock; + private final DeployLogger logger = new SilentDeployLogger(); + + private boolean prepared = false; + + private boolean ignoreLockFailure = false; + private boolean ignoreSessionStaleFailure = false; + + private Deployment(LocalSession session, LocalSessionRepo localSessionRepo, Path tenantPath, ConfigserverConfig configserverConfig, + Optional<Provisioner> hostProvisioner, ActivateLock activateLock, + Duration timeout, Clock clock, boolean prepared) { + this.session = session; + this.localSessionRepo = localSessionRepo; + this.tenantPath = tenantPath; + this.configserverConfig = configserverConfig; + this.hostProvisioner = hostProvisioner; + this.activateLock = activateLock; + this.timeout = timeout; + this.clock = clock; + this.prepared = prepared; + } + + static Deployment unprepared(LocalSession session, LocalSessionRepo localSessionRepo, Path tenantPath, ConfigserverConfig configserverConfig, + Optional<Provisioner> hostProvisioner, ActivateLock activateLock, + Duration timeout, Clock clock) { + return new Deployment(session, localSessionRepo, tenantPath, configserverConfig, hostProvisioner, activateLock, + timeout, clock, false); + } + + static Deployment prepared(LocalSession session, LocalSessionRepo localSessionRepo, + Optional<Provisioner> hostProvisioner, ActivateLock activateLock, + Duration timeout, Clock clock) { + return new Deployment(session, localSessionRepo, null, null, hostProvisioner, activateLock, + timeout, clock, true); + } + + public Deployment setIgnoreLockFailure(boolean ignoreLockFailure) { + this.ignoreLockFailure = ignoreLockFailure; + return this; + } + + public Deployment setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) { + this.ignoreSessionStaleFailure = ignoreSessionStaleFailure; + return this; + } + + /** Prepares this. This does nothing if this is already prepared */ + @Override + public void prepare() { + if (prepared) return; + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + session.prepare(logger, + /** Assumes that session has already set application id, see {@link com.yahoo.vespa.config.server.session.SessionFactoryImpl}. */ + new PrepareParams(configserverConfig).applicationId(session.getApplicationId()).timeoutBudget(timeoutBudget), + Optional.empty(), + tenantPath); + this.prepared = true; + } + + /** Activates this. If it is not already prepared, this will call prepare first. */ + @Override + public void activate() { + if (! prepared) + prepare(); + + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + long sessionId = session.getSessionId(); + validateSessionStatus(session); + try { + activateLock.acquire(timeoutBudget, ignoreLockFailure); + NestedTransaction transaction = new NestedTransaction(); + transaction.add(deactivateCurrentActivateNew(localSessionRepo.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure)); + if (hostProvisioner.isPresent() && !session.getApplicationId().isHostedVespaRoutingApplication()) { + ProvisionInfo info = session.getProvisionInfo(); + hostProvisioner.get().activate(transaction, session.getApplicationId(), info.getHosts()); + } + transaction.commit(); + session.waitUntilActivated(timeoutBudget); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new InternalServerException("Error activating application", e); + } finally { + activateLock.release(); + } + final ApplicationMetaData metaData = session.getMetaData(); + log.log(LogLevel.INFO, session.logPre() + "Session " + sessionId + " activated successfully. Config generation " + metaData.getGeneration()); + } + + /** + * Request a restart of services of this application on hosts matching the filter. + * This is sometimes needed after activation, but can also be requested without + * doing prepare and activate in the same session. + */ + public void restart(HostFilter filter) { + hostProvisioner.get().restart(session.getApplicationId(), filter); + } + + private long validateSessionStatus(LocalSession localSession) { + long sessionId = localSession.getSessionId(); + if (Session.Status.NEW.equals(localSession.getStatus())) { + throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is not prepared"); + } else if (Session.Status.ACTIVATE.equals(localSession.getStatus())) { + throw new IllegalArgumentException(localSession.logPre() + "Session " + sessionId + " is already active"); + } + return sessionId; + } + + private Transaction deactivateCurrentActivateNew(LocalSession currentActiveSession, LocalSession session, boolean ignoreStaleSessionFailure) { + Transaction transaction = session.createActivateTransaction(); + if (isValidSession(currentActiveSession)) { + checkIfActiveHasChanged(session, currentActiveSession, ignoreStaleSessionFailure); + checkIfActiveIsNewerThanSessionToBeActivated(session.getSessionId(), currentActiveSession.getSessionId()); + transaction.add(currentActiveSession.createDeactivateTransaction().operations()); + } + return transaction; + } + + private boolean isValidSession(LocalSession session) { + return session != null; + } + + private void checkIfActiveHasChanged(LocalSession session, LocalSession currentActiveSession, boolean ignoreStaleSessionFailure) { + long activeSessionAtCreate = session.getActiveSessionAtCreate(); + log.log(LogLevel.DEBUG, currentActiveSession.logPre()+"active session id at create time=" + activeSessionAtCreate); + if (activeSessionAtCreate == 0) return; // No active session at create + + long sessionId = session.getSessionId(); + long currentActiveSessionSessionId = currentActiveSession.getSessionId(); + log.log(LogLevel.DEBUG, currentActiveSession.logPre()+"sessionId=" + sessionId + ", current active session=" + currentActiveSessionSessionId); + if (currentActiveSession.isNewerThan(activeSessionAtCreate) && + currentActiveSessionSessionId != sessionId) { + String errMsg = currentActiveSession.logPre()+"Cannot activate session " + + sessionId + " because the currently active session (" + + currentActiveSessionSessionId + ") has changed since session " + sessionId + + " was created (was " + activeSessionAtCreate + " at creation time)"; + if (ignoreStaleSessionFailure) { + log.warning(errMsg+ " (Continuing because of force.)"); + } else { + throw new IllegalStateException(errMsg); + } + } + } + + // As of now, config generation is based on session id, and config generation must be an monotonically + // increasing number + private void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { + if (sessionId < currentActiveSessionId) { + throw new IllegalArgumentException("It is not possible to activate session " + sessionId + + ", because it is older than current active session (" + currentActiveSessionId + ")"); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java new file mode 100644 index 00000000000..9a6f373a807 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.config.model.api.*; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; + +import java.io.File; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Implementation of {@link ModelContext} for configserver. + * + * @author lulf + */ +public class ModelContextImpl implements ModelContext { + private final ApplicationPackage applicationPackage; + private final Optional<Model> previousModel; + private final Optional<ApplicationPackage> permanentApplicationPackage; + private final DeployLogger deployLogger; + private final ConfigDefinitionRepo configDefinitionRepo; + private final FileRegistry fileRegistry; + private final Optional<HostProvisioner> hostProvisioner; + private final ModelContext.Properties properties; + private final Optional<File> appDir; + Optional<Version> vespaVersion; + + public ModelContextImpl(ApplicationPackage applicationPackage, + Optional<Model> previousModel, + Optional<ApplicationPackage> permanentApplicationPackage, + DeployLogger deployLogger, + ConfigDefinitionRepo configDefinitionRepo, + FileRegistry fileRegistry, + Optional<HostProvisioner> hostProvisioner, + ModelContext.Properties properties, + Optional<File> appDir, + Optional<Version> vespaVersion) { + this.applicationPackage = applicationPackage; + this.previousModel = previousModel; + this.permanentApplicationPackage = permanentApplicationPackage; + this.deployLogger = deployLogger; + this.configDefinitionRepo = configDefinitionRepo; + this.fileRegistry = fileRegistry; + this.hostProvisioner = hostProvisioner; + this.properties = properties; + this.appDir = appDir; + this.vespaVersion = vespaVersion; + } + + @Override + public ApplicationPackage applicationPackage() { + return applicationPackage; + } + + @Override + public Optional<Model> previousModel() { + return previousModel; + } + + @Override + public Optional<ApplicationPackage> permanentApplicationPackage() { + return permanentApplicationPackage; + } + + @Override + public Optional<HostProvisioner> hostProvisioner() { + return hostProvisioner; + } + + @Override + public DeployLogger deployLogger() { + return deployLogger; + } + + @Override + public ConfigDefinitionRepo configDefinitionRepo() { + return configDefinitionRepo; + } + + @Override + public FileRegistry getFileRegistry() { + return fileRegistry; + } + + @Override + public ModelContext.Properties properties() { + return properties; + } + + @Override + public Optional<File> appDir() { + return appDir; + } + + @Override + public Optional<Version> vespaVersion() { return vespaVersion; } + + /** + * @author lulf + */ + public static class Properties implements ModelContext.Properties { + private final ApplicationId applicationId; + private final boolean multitenant; + private final List<ConfigServerSpec> configServerSpecs; + private final boolean hostedVespa; + private final Zone zone; + private final Set<Rotation> rotations; + + public Properties(ApplicationId applicationId, + boolean multitenant, + List<ConfigServerSpec> configServerSpecs, + boolean hostedVespa, + Zone zone, + Set<Rotation> rotations) { + this.applicationId = applicationId; + this.multitenant = multitenant; + this.configServerSpecs = configServerSpecs; + this.hostedVespa = hostedVespa; + this.zone = zone; + this.rotations = rotations; + } + + @Override + public boolean multitenant() { + return multitenant; + } + + @Override + public ApplicationId applicationId() { + return applicationId; + } + + @Override + public List<ConfigServerSpec> configServerSpecs() { + return configServerSpecs; + } + + @Override + public boolean hostedVespa() { + return hostedVespa; + } + + @Override + public Zone zone() { + return zone; + } + + @Override + public Set<Rotation> rotations() { + return rotations; + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java new file mode 100644 index 00000000000..336b50351bb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/TenantFileSystemDirs.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.google.common.io.Files; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ConfigServerDB; + +import java.io.File; + +/* + * Holds file system directories for a tenant + * + * @author tonytv + */ +public class TenantFileSystemDirs { + + private final File serverDB; + private final TenantName tenant; + + public TenantFileSystemDirs(File dir, TenantName tenant) { + this.serverDB = dir; + this.tenant = tenant; + ConfigServerDB.cr(path()); + } + + public static TenantFileSystemDirs createTestDirs(TenantName tenantName) { + return new TenantFileSystemDirs(Files.createTempDir(), tenantName); + } + + public File path() { + return new File(serverDB, Path.fromString("tenants").append(tenant.value()).append("sessions").getRelative()); + } + + public File getUserApplicationDir(long generation) { + return new File(path(), String.valueOf(generation)); + } + + public String getPath() { + return serverDB.getPath(); + } + + public void delete() { + IOUtils.recursiveDeleteDir(new File(serverDB, Path.fromString("tenants").append(tenant.value()).getRelative())); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java new file mode 100644 index 00000000000..9c6f21f3511 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -0,0 +1,378 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.Version; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import org.apache.commons.io.IOUtils; + +import java.io.*; +import java.util.*; + +/** + * A class used for reading and writing application data to zookeeper. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 5.1 + */ +public class ZooKeeperClient { + + private final ConfigCurator configCurator; + private final DeployLogger logger; + private final boolean trace; + /* This is the generation that will be used for reading and writing application data. (1 more than last deployed application) */ + private final Path rootPath; + + static final ApplicationFile.PathFilter xmlFilter = new ApplicationFile.PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().endsWith(".xml"); + } + }; + + public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, boolean trace, Path rootPath) { + this.configCurator = configCurator; + this.logger = logger; + this.trace = trace; + this.rootPath = rootPath; + } + + /** + * Sets up basic node structure in ZooKeeper and purges old data. + * This is the first operation on ZK during deploy-application. + * + * We have retries in this method because there have been cases of stray connection loss to ZK, + * even though the user has started the config server. + * + */ + void setupZooKeeper() { + int retries = 5; + try { + while (retries > 0) { + try { + trace("Setting up ZooKeeper nodes for this application"); + createZooKeeperNodes(); + break; + } catch (RuntimeException e) { + logger.log(LogLevel.FINE, "ZK init failed, retrying: " + e); + retries--; + if (retries == 0) { + throw e; + } + Thread.sleep(100); + // Not reconnecting, ZK is supposed to handle that automatically + // as long as the session doesn't expire. We'll see. + } + } + } catch (Exception e) { + throw new IllegalStateException("Unable to initialize vespa model writing to config server(s) " + + System.getProperty("configsources") + "\n" + + "Please ensure that cloudconfig_server is started on the config server node(s), " + + "and check the vespa log for configserver errors. ", e); + } + } + + /** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */ + private void createZooKeeperNodes() { + if (!configCurator.exists(rootPath.getAbsolute())) { + configCurator.createNode(rootPath.getAbsolute()); + } + + for (String subPath : Arrays.asList( + ConfigCurator.DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USERAPP_ZK_SUBPATH, + ZKApplicationPackage.fileRegistryNode)) { + // TODO The replaceFirst below is hackish. + configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", "")); + } + } + + /** + * Feeds def files and user config into ZK. + * + * @param app the application package to feed to zookeeper + */ + void feedZooKeeper(ApplicationPackage app) { + trace("Feeding application config into ZooKeeper"); + // gives lots and lots of debug output: // BasicConfigurator.configure(); + try { + trace("zk operations: " + configCurator.getNumberOfOperations()); + trace("zk operations: " + configCurator.getNumberOfOperations()); + trace("Feeding user def files into ZooKeeper"); + feedZKUserDefs(app); + trace("zk operations: " + configCurator.getNumberOfOperations()); + trace("Feeding application package into ZooKeeper"); + // TODO 1200 zk operations done in the below method + feedZKAppPkg(app); + feedSearchDefinitions(app); + feedZKUserIncludeDirs(app, app.getUserIncludeDirs()); + trace("zk operations: " + configCurator.getNumberOfOperations()); + trace("zk read operations: " + configCurator.getNumberOfReadOperations()); + trace("zk write operations: " + configCurator.getNumberOfWriteOperations()); + trace("Feeding sd from docproc bundle into ZooKeeper"); + trace("zk operations: " + configCurator.getNumberOfOperations()); + trace("Write application metadata into ZooKeeper"); + feedZKAppMetaData(app.getMetaData()); + trace("zk operations: " + configCurator.getNumberOfOperations()); + } catch (Exception e) { + throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + + "Please ensure that cloudconfig_server is started on the config server node(s), " + + "and check the vespa log for configserver errors. ", e); + } + } + + private void feedSearchDefinitions(ApplicationPackage app) throws IOException { + Collection<NamedReader> sds = app.getSearchDefinitions(); + if (sds.isEmpty()) { + return; + } + Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); + configCurator.createNode(zkPath.getAbsolute()); + // Ensures that ranking expressions and other files are also fed. + feedDirZooKeeper(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); + for (NamedReader sd : sds) { + String name = sd.getName(); + Reader reader = sd.getReader(); + String data = com.yahoo.io.IOUtils.readAll(reader); + reader.close(); + configCurator.putData(zkPath.getAbsolute(), name, data); + } + } + + /** + * Puts the application package files into ZK + * + * @param app The application package to use as input. + * @throws java.io.IOException if not able to write to Zookeeper + */ + void feedZKAppPkg(ApplicationPackage app) throws IOException { + ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() { + @Override + public boolean accept(Path path) { + return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX); + } + }; + // Copy app package files and subdirs into zk + // TODO: We should have a way of doing this which doesn't require repeating all the content + feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.HOSTS)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.DEPLOYMENT_FILE.getName())), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.VALIDATION_OVERRIDES.getName())), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + + feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.TEMPLATES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH), + true); + feedDirZooKeeper(app.getFile(ApplicationPackage.RULES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), + srFilter, true); + feedDirZooKeeper(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), + xmlFilter, true); + feedDirZooKeeper(app.getFile(ApplicationPackage.PAGE_TEMPLATES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.PAGE_TEMPLATES_DIR), + xmlFilter, true); + feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.SEARCHCHAINS_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCHCHAINS_DIR), + xmlFilter, true); + feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.DOCPROCCHAINS_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.DOCPROCCHAINS_DIR), + xmlFilter, true); + feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.ROUTINGTABLES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.ROUTINGTABLES_DIR), + xmlFilter, true); + feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.FILES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.FILES_DIR), + true); + } + + private void feedDirZooKeeper(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { + feedDirZooKeeper(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { + @Override + public boolean accept(Path path) { + return true; + } + }, recurse); + } + + private void feedDirZooKeeper(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { + if (!dir.isDirectory()) { + logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper."); + return; + } + for (ApplicationFile file: listFiles(dir, filenameFilter)) { + String name = file.getPath().getName(); + if (name.startsWith(".")) continue; //.svn , .git ... + if ("CVS".equals(name)) continue; + if (file.isDirectory()) { + configCurator.createNode(path.append(name).getAbsolute()); + if (recurse) { + feedDirZooKeeper(file, path.append(name), filenameFilter, recurse); + } + } else { + feedFileZooKeeper(file, path); + } + } + } + + /** + * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories. + */ + private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) { + List<ApplicationFile> rawList = dir.listFiles(); + List<ApplicationFile> ret = new ArrayList<>(); + if (rawList != null) { + for (ApplicationFile f : rawList) { + if (f.isDirectory()) { + ret.add(f); + } else { + if (filter.accept(f.getPath())) { + ret.add(f); + } + } + } + } + return ret; + } + + private void feedFileZooKeeper(ApplicationFile file, Path zkPath) throws IOException { + if (!file.exists()) { + return; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (InputStream inputStream = file.createInputStream()) { + IOUtils.copy(inputStream, baos); + baos.flush(); + configCurator.putData(zkPath.append(file.getPath().getName()).getAbsolute(), baos.toByteArray()); + } + } + + private void feedZKUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { + // User defined include directories + for (String userInclude : userIncludeDirs) { + ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude)); + final List<ApplicationFile> files = dir.listFiles(); + if (files == null || files.isEmpty()) { + configCurator.createNode(getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude).getAbsolute()); + } + feedDirZooKeeper(dir, + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude), + xmlFilter, true); + } + } + + /** + * Feeds all user-defined .def file from the application package into ZooKeeper (both into + * /defconfigs and /userdefconfigs + */ + private void feedZKUserDefs(ApplicationPackage applicationPackage) { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> configDefs = applicationPackage.getAllExistingConfigDefs(); + for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) { + ConfigDefinitionKey key = entry.getKey(); + String contents = entry.getValue().getUnparsedContent(); + feedDefToZookeeper(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + feedDefToZookeeper(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + } + logger.log(LogLevel.FINE, configDefs.size() + " user config definitions"); + } + + private void feedDefToZookeeper(String name, String namespace, String path, String data) { + feedDefToZookeeper(name, namespace, "", path, com.yahoo.text.Utf8.toBytes(data)); + } + + private void feedDefToZookeeper(String name, String namespace, String version, String path, byte[] data) { + configCurator.putDefData( + ("".equals(namespace)) ? name : (namespace + "." + name), + version, + path, + data); + } + + private void feedZKFileRegistry(Version vespaVersion, FileRegistry fileRegistry) { + trace("Feeding file registry data into ZooKeeper"); + String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); + + configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), + vespaVersion.toSerializedForm(), + exportedRegistry); + } + + /** + * Feeds application metadata to zookeeper. Used by vespamodel to create config + * for application metadata (used by ApplicationStatusHandler) + * + * @param metaData The application metadata. + */ + private void feedZKAppMetaData(ApplicationMetaData metaData) { + configCurator.putData(getZooKeeperAppPath(ConfigCurator.META_ZK_PATH).getAbsolute(), metaData.asJsonString()); + } + + void cleanupZooKeeper() { + trace("Exception occurred. Cleaning up ZooKeeper"); + try { + for (String subPath : Arrays.asList( + ConfigCurator.DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USERAPP_ZK_SUBPATH)) { + configCurator.deleteRecurse(getZooKeeperAppPath(null).append(subPath).getAbsolute()); + } + } catch (Exception e) { + logger.log(LogLevel.WARNING, "Could not clean up in zookeeper"); + //Might be called in an exception handler before re-throw, so do not throw here. + } + } + + /** + * Gets a full ZK app path based on id set in Admin object + * + * + * @param trailingPath trailing part of path to be appended to ZK app path + * @return a String with the full ZK application path including trailing path, if set + */ + Path getZooKeeperAppPath(String trailingPath) { + if (trailingPath != null) { + return rootPath.append(trailingPath); + } else { + return rootPath; + } + } + + void trace(String msg) { + if (trace) { + logger.log(LogLevel.FINE, msg); + } + } + + private void feedProvisionInfo(Version version, ProvisionInfo info) throws IOException { + byte[] json = info.toJson(); + configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).append(version.toSerializedForm()).getAbsolute(), json); + } + + public void feedZKFileRegistries(Map<Version, FileRegistry> fileRegistryMap) throws IOException { + for (Map.Entry<Version, FileRegistry> versionFileRegistryEntry : fileRegistryMap.entrySet()) { + feedZKFileRegistry(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue()); + } + } + + public void feedProvisionInfos(Map<Version, ProvisionInfo> provisionInfoMap) throws IOException { + for (Map.Entry<Version, ProvisionInfo> versionProvisionInfoEntry : provisionInfoMap.entrySet()) { + feedProvisionInfo(versionProvisionInfoEntry.getKey(), versionProvisionInfoEntry.getValue()); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java new file mode 100644 index 00000000000..3a92c4a5ebe --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.Version; + +import java.io.IOException; +import java.util.Map; + +/** + * Interface for initializing zookeeper and deploying an application package to zookeeper. + * Initialize must be called before each deploy. + * + * @author lulf + * @since 5.1 + */ +public class ZooKeeperDeployer { + + private final ZooKeeperClient zooKeeperClient; + + public ZooKeeperDeployer(ZooKeeperClient client) { + this.zooKeeperClient = client; + } + + /** + * Deploys an application package to zookeeper. initialize() must be called before calling this method. + * + * @param applicationPackage The application package to persist. + * @param fileRegistryMap The file registries to persist. + * @param provisionInfoMap The provisioning infos to persist. + * @throws IOException if deploying fails + */ + public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, Map<Version, ProvisionInfo> provisionInfoMap) throws IOException { + zooKeeperClient.setupZooKeeper(); + zooKeeperClient.feedZooKeeper(applicationPackage); + zooKeeperClient.feedZKFileRegistries(fileRegistryMap); + zooKeeperClient.feedProvisionInfos(provisionInfoMap); + } + + public void cleanup() { + zooKeeperClient.cleanupZooKeeper(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java new file mode 100644 index 00000000000..2f9c13587fe --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.vespa.filedistribution.FileDistributionManager; + +import java.util.*; + +/** + * Implements invoker of filedistribution using manager with JNI. + * + * @author tonytv + * @author lulf + * @since 5.1.14 + */ +public class FileDBHandler implements FileDistribution { + private final FileDistributionManager manager; + + public FileDBHandler(FileDistributionManager manager) { + this.manager = manager; + } + + @Override + public void sendDeployedFiles(String hostName, Set<FileReference> fileReferences) { + List<String> referencesAsString = new ArrayList<>(); + for (FileReference reference : fileReferences) { + referencesAsString.add(reference.value()); + } + manager.setDeployedFiles(hostName, referencesAsString); + } + + @Override + public void limitSendingOfDeployedFilesTo(Collection<String> hostNames) { + manager.limitSendingOfDeployedFilesTo(hostNames); + } + + @Override + public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { + manager.removeDeploymentsThatHaveDifferentApplicationId(targetHostnames); + } + + @Override + public void reloadDeployFileDistributor() { + manager.reloadDeployFileDistributor(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java new file mode 100644 index 00000000000..58d651ae33a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.net.HostName; +import com.yahoo.vespa.filedistribution.FileDistributionManager; +import com.yahoo.config.model.application.provider.FileReferenceCreator; + +import java.util.*; + +/** + * @author tonytv + */ +public class FileDBRegistry implements FileRegistry { + + private final FileDistributionManager manager; + private List<Entry> entries = new ArrayList<>(); + private final Map<String, FileReference> fileReferenceCache = new HashMap<>(); + + public FileDBRegistry(FileDistributionManager manager) { + this.manager = manager; + } + + @Override + public synchronized FileReference addFile(String relativePath) { + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); + return cachedReference.orElseGet(() -> { + FileReference newRef = FileReferenceCreator.create(manager.addFile(relativePath)); + entries.add(new Entry(relativePath, newRef)); + fileReferenceCache.put(relativePath, newRef); + return newRef; + }); + } + + @Override + public String fileSourceHost() { + return HostName.getLocalhost(); + } + + @Override + public synchronized List<Entry> export() { + return entries; + } + + @Override + public Set<String> allRelativePaths() { + Set<String> ret = new HashSet<>(); + for (Entry entry : entries) { + ret.add(entry.relativePath); + } + return ret; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java new file mode 100644 index 00000000000..1082e0a7f6c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLock.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorLock; +import com.yahoo.vespa.curator.recipes.CuratorLockException; + +import java.time.Clock; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Global filedistribution lock to ensure only one configserver may work on filedistribution. + * The implementation uses a combination of a {@link java.util.concurrent.locks.ReentrantLock} and + * a {@link CuratorLock} to ensure both mutual exclusion within the JVM and + * across JVMs via ZooKeeper. + * + * @author lulf + */ +public class FileDistributionLock implements Lock { + private final Lock processLock; + private final CuratorLock curatorLock; + + public FileDistributionLock(Curator curator, String zkPath) { + this.processLock = new ReentrantLock(); + this.curatorLock = new CuratorLock(curator, zkPath); + } + + @Override + public void lock() { + processLock.lock(); + try { + curatorLock.lock(); + } catch (CuratorLockException e) { + processLock.unlock(); + throw e; + } + } + + @Override + public void lockInterruptibly() throws InterruptedException { + throw new UnsupportedOperationException(); + + } + + @Override + public boolean tryLock() { + if (processLock.tryLock()) { + if (curatorLock.tryLock()) { + return true; + } else { + processLock.unlock(); + return false; + } + } else { + return false; + } + } + + @Override + public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException { + TimeoutBudget budget = new TimeoutBudget(Clock.systemUTC(), Duration.ofMillis(unit.toMillis(timeout))); + if (processLock.tryLock(budget.timeLeft().toMillis(), TimeUnit.MILLISECONDS)) { + if (curatorLock.tryLock(budget.timeLeft().toMillis(), TimeUnit.MILLISECONDS)) { + return true; + } else { + processLock.unlock(); + return false; + } + } else { + return false; + } + } + + @Override + public void unlock() { + try { + curatorLock.unlock(); + } finally { + processLock.unlock(); + } + } + + @Override + public Condition newCondition() { + throw new UnsupportedOperationException(); + } +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java new file mode 100644 index 00000000000..2453381131d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.vespa.filedistribution.FileDistributionManager; + +import java.io.File; +import java.util.concurrent.locks.Lock; + +/** + * Provides file distribution registry and invoker. + * + * @author lulf + * @since 5.1.14 + */ +public class FileDistributionProvider { + + private final FileRegistry fileRegistry; + private final FileDistribution fileDistribution; + + public FileDistributionProvider(File applicationDir, String zooKeepersSpec, String applicationId, Lock fileDistributionLock) { + ensureDirExists(FileDistribution.getDefaultFileDBPath()); + final FileDistributionManager manager = new FileDistributionManager( + FileDistribution.getDefaultFileDBPath(), + applicationDir, + zooKeepersSpec, + applicationId, + fileDistributionLock); + this.fileDistribution = new FileDBHandler(manager); + this.fileRegistry = new FileDBRegistry(manager); + } + + public FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) { + this.fileRegistry = fileRegistry; + this.fileDistribution = fileDistribution; + } + + public FileRegistry getFileRegistry() { + return fileRegistry; + } + + public FileDistribution getFileDistribution() { + return fileDistribution; + } + + private void ensureDirExists(File dir) { + if (!dir.exists()) { + boolean success = dir.mkdirs(); + if (!success) + throw new RuntimeException("Could not create directory " + dir.getPath()); + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java new file mode 100644 index 00000000000..b61a1c8d9bc --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; + +import java.util.Collection; +import java.util.Set; + +/** + * @author lulf + * @since 5.1 + */ +public class MockFileDBHandler implements FileDistribution { + public int sendDeployedFilesCalled = 0; + public int reloadDeployFileDistributorCalled = 0; + public int limitSendingOfDeployedFilesToCalled = 0; + public int removeDeploymentsThatHaveDifferentApplicationIdCalled = 0; + + @Override + public void sendDeployedFiles(String hostName, Set<FileReference> fileReferences) { + sendDeployedFilesCalled++; + } + + @Override + public void reloadDeployFileDistributor() { + reloadDeployFileDistributorCalled++; + } + + @Override + public void limitSendingOfDeployedFilesTo(Collection<String> hostNames) { + limitSendingOfDeployedFilesToCalled++; + } + + @Override + public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) { + removeDeploymentsThatHaveDifferentApplicationIdCalled++; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java new file mode 100644 index 00000000000..588d7c259b6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.model.application.provider.MockFileRegistry; + +/** + * @author lulf + * @since 5.1 + */ +public class MockFileDistributionProvider extends FileDistributionProvider { + + public MockFileDistributionProvider() { + super(new MockFileRegistry(), new MockFileDBHandler()); + } + + public MockFileDBHandler getMockFileDBHandler() { + return (MockFileDBHandler) getFileDistribution(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/BadRequestException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/BadRequestException.java new file mode 100644 index 00000000000..2af55c343a1 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/BadRequestException.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +/** + * Exception that will create a http response with BAD_REQUEST response code (400) + * + * @author musum + * @since 5.1.17 + */ +public class BadRequestException extends RuntimeException { + + public BadRequestException(String message) { + super(message); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java new file mode 100644 index 00000000000..c2d255e1630 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; + +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Requests for content and content status, both for prepared and active sessions, + * are handled by this class. + * + * @author musum + * @since 5.1.15 + */ +public class ContentHandler { + + public HttpResponse get(ContentRequest request) { + ContentRequest.ReturnType returnType = request.getReturnType(); + String urlBase = request.getUrlBase("/content/"); + if (ContentRequest.ReturnType.STATUS.equals(returnType)) { + return status(request, urlBase); + } else { + return content(request, urlBase); + } + } + + public HttpResponse put(ContentRequest request) { + ApplicationFile file = request.getFile(); + if (request.getPath().endsWith("/")) { + createDirectory(request, file); + } else { + createFile(request, file); + } + return createResponse(request); + } + + public HttpResponse delete(ContentRequest request) { + ApplicationFile file = request.getExistingFile(); + deleteFile(file); + return createResponse(request); + } + + private HttpResponse content(ContentRequest request, String urlBase) { + ApplicationFile file = request.getExistingFile(); + if (file.isDirectory()) { + return new SessionContentListResponse(urlBase, listSortedFiles(file, request.getPath(), request.isRecursive())); + } + return new SessionContentReadResponse(file); + } + + private HttpResponse status(ContentRequest request, String urlBase) { + ApplicationFile file = request.getFile(); + if (file.isDirectory()) { + return new SessionContentStatusListResponse(urlBase, listSortedFiles(file, request.getPath(), request.isRecursive())); + } + return new SessionContentStatusResponse(file, urlBase); + } + + private static List<ApplicationFile> listSortedFiles(ApplicationFile file, String path, boolean recursive) { + if (!path.isEmpty() && !path.endsWith("/")) { + return Arrays.asList(file); + } + List<ApplicationFile> files = file.listFiles(recursive); + Collections.sort(files); + return files; + } + + private void createFile(ContentRequest request, ApplicationFile file) { + if (!request.hasRequestBody()) { + throw new BadRequestException("Request must contain body when creating a file"); + } + try { + file.writeFile(new InputStreamReader(request.getData())); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("StatementWithEmptyBody") + private void createDirectory(ContentRequest request, ApplicationFile file) { + if (request.hasRequestBody()) { + // TODO: Enable when we have a good way to check if request contains a body + // return new HttpErrorResponse(HttpResponse.Status.BAD_REQUEST, "Request should not contain a body when creating directories"); + } + file.createDirectory(); + } + + private void deleteFile(ApplicationFile file) { + try { + file.delete(); + } catch (RuntimeException e) { + throw new BadRequestException("File '" + file.getPath() + "' is not an empty directory"); + } + } + + private HttpResponse createResponse(ContentRequest request) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("prepared", request.getUrlBase("/prepared")); + return new SessionResponse(slime, root); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java new file mode 100644 index 00000000000..d71987449bf --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentRequest.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.session.LocalSession; + +import java.io.InputStream; + +/** + * Represents a {@link ContentRequest}, and contains common functionality for content requests for all content handlers. + * + * @author lulf + * @since 5.3 + */ +public abstract class ContentRequest { + private static final String RETURN_QUERY_PROPERTY = "return"; + + enum ReturnType {CONTENT, STATUS} + + private final long sessionId; + private final String path; + private final ApplicationFile file; + private final HttpRequest request; + + protected ContentRequest(HttpRequest request, LocalSession session) { + this.request = request; + this.sessionId = session.getSessionId(); + this.path = getContentPath(request); + this.file = session.getApplicationFile(Path.fromString(path), getApplicationFileMode(request.getMethod())); + } + + private LocalSession.Mode getApplicationFileMode(com.yahoo.jdisc.http.HttpRequest.Method method) { + switch (method) { + case GET: + case OPTIONS: + return LocalSession.Mode.READ; + default: + return LocalSession.Mode.WRITE; + } + } + + ReturnType getReturnType() { + if (request.hasProperty(RETURN_QUERY_PROPERTY)) { + String type = request.getProperty(RETURN_QUERY_PROPERTY); + switch (type) { + case "content": + return ReturnType.CONTENT; + case "status": + return ReturnType.STATUS; + default: + throw new BadRequestException("return=" + type + " is an illegal argument. Only " + + ReturnType.CONTENT.name() + " and " + ReturnType.STATUS.name() + " are allowed"); + } + } else { + return ReturnType.CONTENT; + } + } + + protected abstract String getPathPrefix(); + protected abstract String getContentPath(HttpRequest request); + + String getUrlBase(String appendStr) { + return Utils.getUrlBase(request, getPathPrefix() + appendStr); + } + + boolean isRecursive() { + return request.getBooleanProperty("recursive"); + } + + boolean hasRequestBody() { + return request.getData() != null; + } + + InputStream getData() { + return request.getData(); + } + + + String getPath() { + return path; + } + + ApplicationFile getFile() { + return file; + } + + ApplicationFile getExistingFile() { + if (!file.exists()) { + throw new NotFoundException("Session " + sessionId + " does not contain a file '" + path + "'"); + } + return file; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java new file mode 100644 index 00000000000..04e286ac96f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigRequest.java @@ -0,0 +1,197 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import javax.annotation.Nullable; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.yahoo.collections.Tuple2; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.VespaVersion; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.v2.HttpConfigRequests; +import com.yahoo.vespa.config.server.http.v2.TenantRequest; +import com.yahoo.vespa.config.util.ConfigUtils; + +/** + * A request to get config, bound to tenant and app id. Used by both v1 and v2 of the config REST API. + * + * @author lulf + * @since 5.1 + */ +public class HttpConfigRequest implements GetConfigRequest, TenantRequest { + private static final String DEFAULT_TENANT = "default"; + private static final String HTTP_PROPERTY_NOCACHE = "noCache"; + private final ConfigKey<?> key; + private final ApplicationId appId; + private final boolean noCache; + + private HttpConfigRequest(ConfigKey<?> key, ApplicationId appId, boolean noCache) { + this.key = key; + this.appId = appId; + this.noCache = noCache; + } + + private static ConfigKey<?> fromRequestV1(HttpRequest req) { + BindingMatch<?> bm = Utils.getBindingMatch(req, "http://*/config/v1/*/*"); // see jdisc-bindings.cfg + String conf = bm.group(2); // The port number is implicitly 1, it seems + String cId; + String cName; + String cNamespace; + if (bm.groupCount() >= 4) { + cId = bm.group(3); + } else { + cId = ""; + } + Tuple2<String, String> nns = nameAndNamespace(conf); + cName = nns.first; + cNamespace = nns.second; + return new ConfigKey<>(cName, cId, cNamespace); + } + + public static HttpConfigRequest createFromRequestV1(HttpRequest req) { + return new HttpConfigRequest(fromRequestV1(req), ApplicationId.defaultId(), req.getBooleanProperty(HTTP_PROPERTY_NOCACHE)); + } + + public static HttpConfigRequest createFromRequestV2(HttpRequest req) { + // Four bindings for this: with full app id or only name, with and without config id (like v1) + BindingMatch<?> bm = HttpConfigRequests.getBindingMatch(req, + "http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*", + "http://*/config/v2/tenant/*/application/*/*/*"); + if (bm.groupCount() > 6) return createFromRequestV2FullAppId(req, bm); + return createFromRequestV2SimpleAppId(req, bm); + } + + // The URL pattern with only tenant and application given + private static HttpConfigRequest createFromRequestV2SimpleAppId(HttpRequest req, BindingMatch<?> bm) { + String cId; + String cName; + String cNamespace; + TenantName tenant = TenantName.from(bm.group(2)); + ApplicationName application = ApplicationName.from(bm.group(3)); + String conf = bm.group(4); + if (bm.groupCount() >= 6) { + cId = bm.group(5); + } else { + cId = ""; + } + Tuple2<String, String> nns = nameAndNamespace(conf); + cName = nns.first; + cNamespace = nns.second; + return new HttpConfigRequest(new ConfigKey<>(cName, cId, cNamespace), + new ApplicationId.Builder().applicationName(application).tenant(tenant).build(), + req.getBooleanProperty(HTTP_PROPERTY_NOCACHE)); + } + + // The URL pattern with full app id given + private static HttpConfigRequest createFromRequestV2FullAppId(HttpRequest req, BindingMatch<?> bm) { + String cId; + String cName; + String cNamespace; + String tenant = bm.group(2); + String application = bm.group(3); + String environment = bm.group(4); + String region = bm.group(5); + String instance = bm.group(6); + String conf = bm.group(7); + if (bm.groupCount() >= 9) { + cId = bm.group(8); + } else { + cId = ""; + } + Tuple2<String, String> nns = nameAndNamespace(conf); + cName = nns.first; + cNamespace = nns.second; + + ApplicationId appId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName(application) + .instanceName(instance) + .build(); + return new HttpConfigRequest(new ConfigKey<>(cName, cId, cNamespace), appId, req.getBooleanProperty(HTTP_PROPERTY_NOCACHE)); + } + + /** + * Throws an exception if bad config or config id + * + * @param requestKey a {@link com.yahoo.vespa.config.ConfigKey} + * @param requestHandler a {@link RequestHandler} + * @param appId appId + */ + public static void validateRequestKey(ConfigKey<?> requestKey, RequestHandler requestHandler, ApplicationId appId) { + Set<ConfigKey<?>> allConfigsProduced = requestHandler.allConfigsProduced(appId, Optional.empty()); + if (allConfigsProduced.isEmpty()) { + // This will happen if the configserver is starting up, but has not built a config model + throwModelNotReady(); + } + if (configNameNotFound(requestKey, allConfigsProduced)) { + throw new NotFoundException("No such config: " + requestKey.getNamespace() + "." + requestKey.getName()); + } + if (configIdNotFound(requestHandler, requestKey, appId)) { + throw new NotFoundException("No such config id: " + requestKey.getConfigId()); + } + } + + public static void throwModelNotReady() { + throw new NotFoundException("Config not available, verify that an application package has been deployed and activated."); + } + + /** + * If the given config is produced by the model at all + * + * @return ok or not + */ + private static boolean configNameNotFound(final ConfigKey<?> requestKey, Set<ConfigKey<?>> allConfigsProduced) { + return !Iterables.any(allConfigsProduced, new Predicate<ConfigKey<?>>() { + @Override + public boolean apply(@Nullable ConfigKey<?> k) { + return k.getName().equals(requestKey.getName()) && k.getNamespace().equals(requestKey.getNamespace()); + } + }); + } + + private static boolean configIdNotFound(RequestHandler requestHandler, ConfigKey<?> requestKey, ApplicationId appId) { + return !requestHandler.allConfigIds(appId, Optional.empty()).contains(requestKey.getConfigId()); + } + + public static Tuple2<String, String> nameAndNamespace(String nsDotName) { + Tuple2<String, String> ret = ConfigUtils.getNameAndNamespaceFromString(nsDotName); + if ("".equals(ret.second)) throw new IllegalArgumentException("Illegal config, must be of form namespace.name."); + return ret; + } + + @Override + public ConfigKey<?> getConfigKey() { + return key; + } + + @Override + public DefContent getDefContent() { + return DefContent.fromList(Collections.<String>emptyList()); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return Optional.empty(); + } + + @Override + public ApplicationId getApplicationId() { + return appId; + } + + public boolean noCache() { + return noCache; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigResponse.java new file mode 100644 index 00000000000..10f5243cfdb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpConfigResponse.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; + +import java.io.IOException; +import java.io.OutputStream; + +import static com.yahoo.jdisc.http.HttpResponse.Status.OK; + +/** + * HTTP getConfig response + * + * @author lulf + * @since 5.1 + */ +public class HttpConfigResponse extends HttpResponse { + public static final String JSON_CONTENT_TYPE = "application/json"; + private final ConfigResponse config; + + private HttpConfigResponse(ConfigResponse config) { + super(OK); + this.config = config; + } + + public static HttpConfigResponse createFromConfig(ConfigResponse config) { + return new HttpConfigResponse(config); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + config.serialize(outputStream, CompressionType.UNCOMPRESSED); + } + + @Override + public String getContentType() { + return JSON_CONTENT_TYPE; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java new file mode 100644 index 00000000000..b5886992f10 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.logging.Logger; + +import static com.yahoo.jdisc.Response.Status.*; + +/** + * @author lulf + * @since 5.1 + */ +public class HttpErrorResponse extends HttpResponse { + Logger log = Logger.getLogger(HttpErrorResponse.class.getName()); + private final Slime slime = new Slime(); + + public HttpErrorResponse(int code, final String errorType, final String msg) { + super(code); + final Cursor root = slime.setObject(); + root.setString("error-code", errorType); + root.setString("message", msg); + if (code != 200) { + log.log(LogLevel.INFO, "Returning response with response code " + code + ", error-code:" + errorType + ", message=" + msg); + } + } + + public enum errorCodes { + NOT_FOUND, + BAD_REQUEST, + METHOD_NOT_ALLOWED, + INTERNAL_SERVER_ERROR, + INVALID_APPLICATION_PACKAGE, + UNKNOWN_VESPA_VERSION, + OUT_OF_CAPACITY + } + + public static HttpErrorResponse notFoundError(String msg) { + return new HttpErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), msg); + } + + public static HttpErrorResponse internalServerError(String msg) { + return new HttpErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), msg); + } + + public static HttpErrorResponse invalidApplicationPackage(String msg) { + return new HttpErrorResponse(BAD_REQUEST, errorCodes.INVALID_APPLICATION_PACKAGE.name(), msg); + } + + public static HttpErrorResponse outOfCapacity(String msg) { + return new HttpErrorResponse(BAD_REQUEST, errorCodes.OUT_OF_CAPACITY.name(), msg); + } + + public static HttpErrorResponse badRequest(String msg) { + return new HttpErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), msg); + } + + public static HttpErrorResponse methodNotAllowed(String msg) { + return new HttpErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), msg); + } + + public static HttpResponse unknownVespaVersion(String message) { + return new HttpErrorResponse(BAD_REQUEST, errorCodes.UNKNOWN_VESPA_VERSION.name(), message); + } + + @Override + public void render(OutputStream stream) throws IOException { + new JsonFormat(true).encode(stream, slime); + } + + //@Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java new file mode 100644 index 00000000000..50a1a3877ef --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.config.provision.ApplicationId; + +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * HTTP handler for a v2 getConfig operation + * + * @author lulf + * @since 5.1 + */ +public class HttpGetConfigHandler extends HttpHandler { + private final RequestHandler requestHandler; + + public HttpGetConfigHandler(Executor executor, RequestHandler requestHandler, AccessLog accessLog) { + super(executor, accessLog); + this.requestHandler = requestHandler; + } + + @Inject + public HttpGetConfigHandler(Executor executor, Tenants tenants, AccessLog accesslog) { + this(executor, tenants.defaultTenant().getRequestHandler(), accesslog); + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + HttpConfigRequest request = HttpConfigRequest.createFromRequestV1(req); + HttpConfigRequest.validateRequestKey(request.getConfigKey(), requestHandler, ApplicationId.defaultId()); + return HttpConfigResponse.createFromConfig(resolveConfig(request)); + } + + private ConfigResponse resolveConfig(HttpConfigRequest request) { + log.log(LogLevel.DEBUG, "nocache=" + request.noCache()); + ConfigResponse config = requestHandler.resolveConfig(ApplicationId.defaultId(), request, Optional.empty()); + if (config == null) HttpConfigRequest.throwModelNotReady(); + return config; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java new file mode 100644 index 00000000000..7eb6f9c2271 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.log.LogLevel; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.yolean.Exceptions; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.concurrent.Executor; + +/** + * Super class for http handlers, that takes care of checking valid + * methods for a request. Handlers should subclass this method and + * implement the handleMETHOD methods that it supports. + * + * @author musum + * @since 5.1.14 + */ +public class HttpHandler extends LoggingRequestHandler { + + public HttpHandler(Executor executor, AccessLog accessLog) { + super(executor, accessLog); + } + + @Override + public HttpResponse handle(HttpRequest request) { + log.log(LogLevel.DEBUG, request.getMethod() + " " + request.getUri().toString()); + try { + switch (request.getMethod()) { + case POST: + return handlePOST(request); + case GET: + return handleGET(request); + case PUT: + return handlePUT(request); + case DELETE: + return handleDELETE(request); + default: + return createErrorResponse(request.getMethod()); + } + } catch (NotFoundException e) { + return HttpErrorResponse.notFoundError(getMessage(e, request)); + } catch (BadRequestException e) { + return HttpErrorResponse.badRequest(getMessage(e, request)); + } catch (IllegalArgumentException | IllegalStateException e) { + return HttpErrorResponse.badRequest(getMessage(e, request)); + } catch (InvalidApplicationException e) { + return HttpErrorResponse.invalidApplicationPackage(getMessage(e, request)); + } catch (OutOfCapacityException e) { + return HttpErrorResponse.outOfCapacity(getMessage(e, request)); + } catch (InternalServerException e) { + return HttpErrorResponse.internalServerError(getMessage(e, request)); + } catch (UnknownVespaVersionException e) { + return HttpErrorResponse.unknownVespaVersion(getMessage(e, request)); + } catch (Exception e) { + e.printStackTrace(); + return HttpErrorResponse.internalServerError(getMessage(e, request)); + } + } + + private String getMessage(Exception e, HttpRequest request) { + String message; + if (request.getBooleanProperty("debug")) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + message = sw.toString(); + } else { + message = Exceptions.toMessageString(e); + } + return message; + } + + /** + * Default implementation of handler for GET requests. Returns an error response. + * Override this method to handle GET requests. + * + * @param request a {@link HttpRequest} + * @return an error response with response code 405 + */ + protected HttpResponse handleGET(HttpRequest request) { + return createErrorResponse(request.getMethod()); + } + + /** + * Default implementation of handler for POST requests. Returns an error response. + * Override this method to handle POST requests. + * + * @param request a {@link HttpRequest} + * @return an error response with response code 405 + */ + protected HttpResponse handlePOST(HttpRequest request) { + return createErrorResponse(request.getMethod()); + } + + /** + * Default implementation of handler for PUT requests. Returns an error response. + * Override this method to handle POST requests. + * + * @param request a {@link HttpRequest} + * @return an error response with response code 405 + */ + protected HttpResponse handlePUT(HttpRequest request) { + return createErrorResponse(request.getMethod()); + } + + /** + * Default implementation of handler for DELETE requests. Returns an error response. + * Override this method to handle DELETE requests. + * + * @param request a {@link HttpRequest} + * @return an error response with response code 405 + */ + protected HttpResponse handleDELETE(HttpRequest request) { + return createErrorResponse(request.getMethod()); + } + + /** + * Creates error response when method is not handled + * + * @return an error response with response code 405 + */ + private HttpResponse createErrorResponse(com.yahoo.jdisc.http.HttpRequest.Method method) { + return HttpErrorResponse.methodNotAllowed("Method '" + method + "' is not supported"); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java new file mode 100644 index 00000000000..a6e2c5bf050 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandler.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.config.provision.ApplicationId; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.http.HttpResponse.Status.*; + + +/** + * Handler for a list configs operation. Lists all configs in model. + * + * @author vegardh + * @since 5.1.11 + */ +public class HttpListConfigsHandler extends HttpHandler { + static final String RECURSIVE_QUERY_PROPERTY = "recursive"; + private final RequestHandler requestHandler; + + @Inject + public HttpListConfigsHandler(Executor executor, AccessLog accessLog, Tenants tenants) { + this(executor, accessLog, tenants.defaultTenant().getRequestHandler()); + } + + public HttpListConfigsHandler(Executor executor, AccessLog accessLog, RequestHandler requestHandler) { + super(executor, accessLog); + this.requestHandler = requestHandler; + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + boolean recursive = req.getBooleanProperty(RECURSIVE_QUERY_PROPERTY); + Set<ConfigKey<?>> configs = requestHandler.listConfigs(ApplicationId.defaultId(), Optional.empty(), recursive); + String urlBase = Utils.getUrlBase(req, "/config/v1/"); + Set<ConfigKey<?>> allConfigs = requestHandler.allConfigsProduced(ApplicationId.defaultId(), Optional.empty()); + return new ListConfigsResponse(configs, allConfigs, urlBase, recursive); + } + + static class ListConfigsResponse extends HttpResponse { + private final List<ConfigKey<?>> configs; + private final Set<ConfigKey<?>> allConfigs; + private final String urlBase; + private final boolean recursive; + + /** + * New list response + * + * @param configs the configs to include in the list + * @param urlBase for example "http://foo.com:19071/config/v1/ (configs are appended to the listed URLs based on configs list) + * @param recursive list recursively + */ + public ListConfigsResponse(Set<ConfigKey<?>> configs, Set<ConfigKey<?>> allConfigs, String urlBase, boolean recursive) { + super(OK); + this.configs = new ArrayList<>(configs); + Collections.sort(this.configs); + this.allConfigs = allConfigs; + this.urlBase = urlBase; + this.recursive = recursive; + } + + /** + * The listing URL for this config in this service + * + * @param key config key + * @param rec recursive + * @return url + */ + String toUrl(ConfigKey<?> key, boolean rec) { + return urlBase + key.getNamespace() + "." + key.getName() + "/" + key.getConfigId() + (rec ? "" : "/"); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor array; + if (!recursive) { + array = root.setArray("children"); + for (ConfigKey<?> key : keysThatHaveAChildWithSameName(configs, allConfigs)) { + array.addString(toUrl(key, false)); + } + } + array = root.setArray("configs"); + for (ConfigKey<?> key : configs) { + array.addString(toUrl(key, true)); + } + new JsonFormat(true).encode(outputStream, slime); + } + + static Set<ConfigKey<?>> keysThatHaveAChildWithSameName(Collection<ConfigKey<?>> keys, Set<ConfigKey<?>> allConfigs) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> k : keys) { + if (ListConfigsResponse.hasAChild(k, allConfigs)) ret.add(k); + } + return ret; + } + + static boolean hasAChild(ConfigKey<?> key, Set<ConfigKey<?>> keys) { + if ("".equals(key.getConfigId())) return false; + for (ConfigKey<?> k : keys) { + if (!k.getName().equals(key.getName())) continue; + if ("".equals(k.getConfigId())) continue; + if (k.getConfigId().equals(key.getConfigId())) continue; + if (k.getConfigId().startsWith(key.getConfigId())) return true; + } + return false; + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java new file mode 100644 index 00000000000..31d6e84860d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpListNamedConfigsHandler.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.google.inject.Inject; +import com.yahoo.collections.Tuple2; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.config.provision.ApplicationId; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executor; + +/** + * Handler for a list configs of given name operation. Lists all configs in model for a given config name. + * + * @author vegardh + * @since 5.1.11 + */ +public class HttpListNamedConfigsHandler extends HttpHandler { + private final RequestHandler requestHandler; + + public HttpListNamedConfigsHandler(Executor executor, RequestHandler requestHandler, AccessLog accessLog) { + super(executor, accessLog); + this.requestHandler = requestHandler; + } + + @Inject + public HttpListNamedConfigsHandler(Executor executor, Tenants tenants, AccessLog accessLog) { + this(executor, tenants.defaultTenant().getRequestHandler(), accessLog); + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + boolean recursive = req.getBooleanProperty(HttpListConfigsHandler.RECURSIVE_QUERY_PROPERTY); + ConfigKey<?> listKey = parseReqToKey(req); + HttpConfigRequest.validateRequestKey(listKey, requestHandler, ApplicationId.defaultId()); + Set<ConfigKey<?>> configs = requestHandler.listNamedConfigs(ApplicationId.defaultId(), Optional.empty(), listKey, recursive); + String urlBase = Utils.getUrlBase(req, "/config/v1/"); + return new HttpListConfigsHandler.ListConfigsResponse(configs, requestHandler.allConfigsProduced(ApplicationId.defaultId(), Optional.empty()), urlBase, recursive); + } + + private ConfigKey<?> parseReqToKey(HttpRequest req) { + BindingMatch<?> bm = Utils.getBindingMatch(req, "http://*/config/v1/*/*"); + String config = bm.group(2); // See jdisc-bindings.cfg. The port number is implicitly 1, it seems. + Tuple2<String, String> nns = HttpConfigRequest.nameAndNamespace(config); + String name = nns.first; + String namespace = nns.second; + String idSegment = ""; + if (bm.groupCount() == 4) { + idSegment = bm.group(3); + } + return new ConfigKey<>(name, idSegment, namespace); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/InternalServerException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/InternalServerException.java new file mode 100644 index 00000000000..240f5814652 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/InternalServerException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import java.io.IOException; + +/** + * Exception that will create a http response with INTERNAL_SERVER_ERROR response code (500) + * + * @author musum + * @since 5.1.17 + */ +public class InternalServerException extends RuntimeException { + + public InternalServerException(String message) { + super(message); + } + + public InternalServerException(String message, Exception e) { + super(message, e); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/InvalidApplicationException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/InvalidApplicationException.java new file mode 100644 index 00000000000..ba8f034777a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/InvalidApplicationException.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +/** + * @author musum + */ +public class InvalidApplicationException extends RuntimeException { + + public InvalidApplicationException(String message) { + super(message); + } + + public InvalidApplicationException(String message, Throwable e) { + super(message, e); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/JSONResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/JSONResponse.java new file mode 100644 index 00000000000..ae99481ccfa --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/JSONResponse.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Response that contains some utility stuff for rendering json. + * + * @author lulf + * @since 5.3 + */ +public class JSONResponse extends HttpResponse { + private final Slime slime = new Slime(); + protected final Cursor object; + public JSONResponse(int status) { + super(status); + this.object = slime.setObject(); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java new file mode 100644 index 00000000000..327e792134a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/NotFoundException.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +/** + * Exception that will create a http response with NOT_FOUND response code (404) + * + * @author musum + * @since 5.1.17 + */ +public class NotFoundException extends RuntimeException { + + public NotFoundException(String message) { + super(message); + } +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java new file mode 100644 index 00000000000..53133b49c8a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.provision.Provisioner; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.config.server.ActivateLock; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.deploy.Deployer; +import com.yahoo.vespa.config.server.deploy.Deployment; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; + +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * @author lulf + */ +public class SessionActiveHandlerBase extends SessionHandler { + + public SessionActiveHandlerBase(Executor executor, AccessLog accessLog) { + super(executor, accessLog); + } + + protected void activate(HttpRequest request, + LocalSessionRepo localSessionRepo, + ActivateLock activateLock, + TimeoutBudget timeoutBudget, + Optional<Provisioner> hostProvisioner, + LocalSession localSession) { + // TODO: Use an injected deployer from the callers of this instead + // TODO: And then get rid of the activateLock and localSessionRepo arguments in deployFromPreparedSession + Deployer deployer = new Deployer(null, HostProvisionerProvider.from(hostProvisioner), null, null); + Deployment deployment = deployer.deployFromPreparedSession(localSession, activateLock, localSessionRepo, timeoutBudget.timeLeft()); + deployment.setIgnoreLockFailure(shouldIgnoreLockFailure(request)); + deployment.setIgnoreSessionStaleFailure(shouldIgnoreSessionStaleFailure(request)); + deployment.activate(); + } + + private boolean shouldIgnoreLockFailure(HttpRequest request) { + return request.getBooleanProperty("force"); + } + + /** + * True if this request should ignore activation failure because the session was made from an active session that is not active now + * @param request a {@link com.yahoo.container.jdisc.HttpRequest} + * @return true if ignore failure + */ + private boolean shouldIgnoreSessionStaleFailure(HttpRequest request) { + return request.getBooleanProperty("force"); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java new file mode 100644 index 00000000000..90b5a7fd869 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; + +/** + * Represents a request for listing files within an application package. + * + * @author lulf + * @since 5.1 + */ +class SessionContentListResponse extends SessionResponse { + private final Slime slime = new Slime(); + + public SessionContentListResponse(String urlBase, List<ApplicationFile> files) { + super(); + Cursor array = slime.setArray(); + for (ApplicationFile file : files) { + array.addString(urlBase + file.getPath() + (file.isDirectory() ? "/" : "")); + } + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java new file mode 100644 index 00000000000..045251d0623 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentReadResponse.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.container.jdisc.HttpResponse; +import org.apache.commons.io.IOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import static com.yahoo.jdisc.http.HttpResponse.Status.*; + +/** + * Represents a response for a request to read contents of a file. + * + * @author lulf + * @since 5.1 + */ +public class SessionContentReadResponse extends HttpResponse { + private final ApplicationFile file; + + public SessionContentReadResponse(ApplicationFile file) { + super(OK); + this.file = file; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + try (InputStream inputStream = file.createInputStream()) { + IOUtils.copyLarge(inputStream, outputStream, new byte[1]); + } + } + + @Override + public String getContentType() { + return HttpResponse.DEFAULT_MIME_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java new file mode 100644 index 00000000000..caf38517b6a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; + +/** + * Status and md5sum for files within an application package. + * + * @author musum + * @since 5.1.15 + */ +class SessionContentStatusListResponse extends SessionResponse { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger("SessionContentStatusListResponse"); + + private final Slime slime = new Slime(); + + public SessionContentStatusListResponse(String urlBase, List<ApplicationFile> files) { + super(); + Cursor array = slime.setArray(); + for (ApplicationFile f : files) { + Cursor element = array.addObject(); + element.setString("status", f.getMetaData().getStatus()); + element.setString("md5", f.getMetaData().getMd5()); + element.setString("name", urlBase + f.getPath()); + log.log(LogLevel.DEBUG, "Adding file " + urlBase + f.getPath()); + } + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java new file mode 100644 index 00000000000..15c852b66c3 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.application.api.ApplicationFile; + +import java.io.*; + +/** + * Represents a response for a request to show the status and md5sum of a file in the application package. + * + * @author musum + * @since 5.1.15 + */ +public class SessionContentStatusResponse extends SessionResponse { + private final ApplicationFile file; + private final String urlBase; + private final ApplicationFile.MetaData metaData; + private final ObjectMapper mapper = new ObjectMapper(); + + public SessionContentStatusResponse(ApplicationFile file, String urlBase) { + super(); + this.file = file; + this.urlBase = urlBase; + + ApplicationFile.MetaData metaData; + if (file == null) { + metaData = new ApplicationFile.MetaData(ApplicationFile.ContentStatusDeleted, ""); + } else { + metaData = file.getMetaData(); + } + if (metaData == null) { + throw new IllegalArgumentException("Could not find status for '" + file.getPath() + "'"); + } + this.metaData = metaData; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + mapper.writeValue(outputStream, new ResponseData(metaData.status, metaData.md5, urlBase + file.getPath())); + } + + private static class ResponseData { + public final String status; + public final String md5; + public final String name; + + private ResponseData(String status, String md5, String name) { + this.status = status; + this.md5 = md5; + this.name = name; + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionCreate.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionCreate.java new file mode 100644 index 00000000000..1526a5b4e0e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionCreate.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.google.common.io.Files; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.CompressedApplicationInputStream; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.SessionFactory; + +import java.io.File; +import java.io.IOException; + +/** + * Creates a session from an application package, + * or creates a new session from a previous session (with id or the "active" session). + * Handles /application/v2 requests + * + * @author lulf + * @author musum + * @since 5.1.27 + */ +// TODO Rename class +public class SessionCreate { + public final static String APPLICATION_X_GZIP = "application/x-gzip"; + public final static String APPLICATION_ZIP = "application/zip"; + public final static String contentTypeHeader = "Content-Type"; + + private final SessionFactory sessionFactory; + private final LocalSessionRepo localSessionRepo; + private final SessionCreateResponse responseCreator; + + public SessionCreate(SessionFactory sessionFactory, LocalSessionRepo localSessionRepo, SessionCreateResponse responseCreator) { + this.sessionFactory = sessionFactory; + this.localSessionRepo = localSessionRepo; + this.responseCreator = responseCreator; + } + + public HttpResponse createFromExisting(HttpRequest request, Slime deployLog, LocalSession fromSession, TenantName tenant, TimeoutBudget timeoutBudget) { + DeployLogger logger = SessionHandler.createLogger(deployLog, request, + new ApplicationId.Builder().tenant(tenant).applicationName("-").build()); + LocalSession session = sessionFactory.createSessionFromExisting(fromSession, logger, timeoutBudget); + localSessionRepo.addSession(session); + return createResponse(request, session); + } + + public HttpResponse create(HttpRequest request, Slime deployLog, TenantName tenant, TimeoutBudget timeoutBudget) { + validateDataAndHeader(request); + return createSession(request, deployLog, sessionFactory, localSessionRepo, tenant, timeoutBudget); + } + + private HttpResponse createSession(HttpRequest request, Slime deployLog, SessionFactory sessionFactory, LocalSessionRepo localSessionRepo, TenantName tenant, TimeoutBudget timeoutBudget) { + File tempDir = Files.createTempDir(); + File applicationDirectory = decompressApplication(request, tempDir); + DeployLogger logger = SessionHandler.createLogger(deployLog, request, + new ApplicationId.Builder().tenant(tenant).applicationName("-").build()); + String name = getNameProperty(request, logger); + LocalSession session = sessionFactory.createSession(applicationDirectory, name, logger, timeoutBudget); + localSessionRepo.addSession(session); + HttpResponse response = createResponse(request, session); + cleanupApplicationDirectory(tempDir, logger); + return response; + } + + private String getNameProperty(HttpRequest request, DeployLogger logger) { + String name = request.getProperty("name"); + // TODO: Do we need validation of this parameter? + if (name == null) { + name = "default"; + logger.log(LogLevel.INFO, "No application name given, using '" + name + "'"); + } + return name; + } + + private File decompressApplication(HttpRequest request, File tempDir) { + try (CompressedApplicationInputStream application = CompressedApplicationInputStream.createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader))) { + return application.decompress(tempDir); + } catch (IOException e) { + throw new InternalServerException("Unable to decompress data in body", e); + } + } + + private void cleanupApplicationDirectory(File tempDir, DeployLogger logger) { + logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); + if (!IOUtils.recursiveDeleteDir(tempDir)) { + logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); + } + } + + + private static void validateDataAndHeader(HttpRequest request) { + if (request.getData() == null) { + throw new BadRequestException("Request contains no data"); + } + String header = request.getHeader(contentTypeHeader); + if (header == null) { + throw new BadRequestException("Request contains no " + contentTypeHeader + " header"); + } else if (!(header.equals(APPLICATION_X_GZIP) || header.equals(APPLICATION_ZIP))) { + throw new BadRequestException("Request contains invalid " + contentTypeHeader + " header, only '" + + APPLICATION_X_GZIP + "' and '" + APPLICATION_ZIP + "' are supported"); + } + } + + private HttpResponse createResponse(HttpRequest request, LocalSession session) { + return responseCreator.createResponse(request.getHost(), request.getPort(), session.getSessionId()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionCreateResponse.java new file mode 100644 index 00000000000..72810ed394c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionCreateResponse.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; + +/** + * Interface for creating responses for SessionCreateHandler. + * + * @author musum + * @since 5.1.27 + */ +public interface SessionCreateResponse { + + public HttpResponse createResponse(String hostName, int port, long sessionId); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java new file mode 100644 index 00000000000..c8829cf9e4d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.DeployHandlerLogger; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.session.Session; +import com.yahoo.vespa.config.server.session.SessionRepo; + +import java.time.Clock; +import java.time.Duration; +import java.util.concurrent.Executor; + + +/** + * Super class for session handlers, that takes care of checking valid + * methods for a request. Session handlers should subclass this method and + * implement the handleMETHOD methods that it supports. + * + * @author musum + * @since 5.1.14 + */ +public class SessionHandler extends HttpHandler { + public SessionHandler(Executor executor, AccessLog accessLog) { + super(executor, accessLog); + } + + /** + * Gets the raw session id from request (v2). Input request must have a valid path. + * + * @param request a request + * @return a session id + */ + public static String getRawSessionIdV2(HttpRequest request) { + final String path = request.getUri().toString(); + BindingMatch<?> bm = Utils.getBindingMatch(request, "http://*/application/v2/tenant/*/session/*/*"); + if (bm.groupCount() < 4) { + // This would mean the subtype of this doesn't have the correct binding + throw new IllegalArgumentException("Can not get session id from request '" + path + "'"); + } + return bm.group(3); + } + + /** + * Gets session id (as a number) from request (v2). Input request must have a valid path. + * + * @param request a request + * @return a session id + */ + static Long getSessionIdV2(HttpRequest request) { + try { + return Long.parseLong(getRawSessionIdV2(request)); + } catch (NumberFormatException e) { + throw createSessionException(request); + } + } + + private static BadRequestException createSessionException(HttpRequest request) { + return new BadRequestException("Session id in request is not a number, request was '" + + request.getUri().toString() + "'"); + } + + public static <SESSIONTYPE extends Session> SESSIONTYPE getSessionFromRequestV2(SessionRepo<SESSIONTYPE> sessionRepo, HttpRequest request) { + long sessionId = getSessionIdV2(request); + return getSessionFromRequest(sessionRepo, sessionId); + } + + public static <SESSIONTYPE extends Session> SESSIONTYPE getSessionFromRequest(SessionRepo<SESSIONTYPE> sessionRepo, long sessionId) { + SESSIONTYPE session = sessionRepo.getSession(sessionId); + if (session == null) { + throw new NotFoundException("Session " + sessionId + " was not found"); + } + return session; + } + + protected static final Duration DEFAULT_ACTIVATE_TIMEOUT = Duration.ofMinutes(2); + + public static TimeoutBudget getTimeoutBudget(HttpRequest request, Duration defaultTimeout) { + return new TimeoutBudget(Clock.systemUTC(), getRequestTimeout(request, defaultTimeout)); + } + + + protected static Duration getRequestTimeout(HttpRequest request, Duration defaultTimeout) { + if (!request.hasProperty("timeout")) { + return defaultTimeout; + } + try { + return Duration.ofSeconds((long) Double.parseDouble(request.getProperty("timeout"))); + } catch (Exception e) { + return defaultTimeout; + } + } + + static DeployHandlerLogger createLogger(Slime deployLog, HttpRequest request, ApplicationId app) { + return createLogger(deployLog, request.getBooleanProperty("verbose"), app); + } + + public static DeployHandlerLogger createLogger(Slime deployLog, boolean verbose, ApplicationId app) { + return new DeployHandlerLogger(deployLog.get().setArray("log"), verbose, app); + } + + protected Slime createDeployLog() { + Slime deployLog = new Slime(); + deployLog.setObject(); + return deployLog; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java new file mode 100644 index 00000000000..5054e1129d1 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import java.io.IOException; +import java.io.OutputStream; + +import static com.yahoo.jdisc.http.HttpResponse.Status.OK; + +/** + * Superclass for responses from session HTTP handlers. Implements the + * render method. + * + * @author musum + * @since 5.1.14 + */ +public class SessionResponse extends HttpResponse { + private final Slime slime; + protected final Cursor root; + + public SessionResponse() { + super(OK); + slime = new Slime(); + root = slime.setObject(); + } + + public SessionResponse(Slime slime, Cursor root) { + super(OK); + this.slime = slime; + this.root = root; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/UnknownVespaVersionException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/UnknownVespaVersionException.java new file mode 100644 index 00000000000..bfdbdd1d4b1 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/UnknownVespaVersionException.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +/** + * @author musum + * @since 5.39 + */ +public class UnknownVespaVersionException extends RuntimeException { + + public UnknownVespaVersionException(String message) { + super(message); + } + + public UnknownVespaVersionException(String message, Throwable e) { + super(message, e); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java new file mode 100644 index 00000000000..83f4c836d20 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/Utils.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.jdisc.application.UriPattern; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; + +import java.net.URI; + +/** + * Utilities for handlers. + * + * @author musum + * @since 5.1.14 + */ +public class Utils { + + /** + * If request is an HTTP request and a jdisc request, return the {@link com.yahoo.jdisc.application.BindingMatch} + * of the request. Otherwise return a dummy match useful only for testing based on the <code>uriPattern</code> + * supplied. + * + * @param req an {@link com.yahoo.container.jdisc.HttpRequest} + * @param uriPattern a pattern to create a BindingMatch for in tests + * @return match + */ + public static BindingMatch<?> getBindingMatch(HttpRequest req, String uriPattern) { + com.yahoo.jdisc.http.HttpRequest jDiscRequest = req.getJDiscRequest(); + BindingMatch<?> bm = jDiscRequest.getBindingMatch(); + if (bm == null) { + bm = new BindingMatch<>( + new UriPattern(uriPattern).match(URI.create(jDiscRequest.getUri().toString())), + new Object()); + } + return bm; + } + + public static String getUrlBase(HttpRequest request, String pathPrefix) { + return request.getUri().getScheme() + "://" + request.getHost() + ":" + request.getPort() + pathPrefix; + } + + public static Tenant checkThatTenantExists(Tenants tenants, TenantName tenant) { + if (!tenants.tenantsCopy().containsKey(tenant)) { + throw new NotFoundException("Tenant '" + tenant + "' was not found."); + } + return tenants.tenantsCopy().get(tenant); + } + + public static void checkThatTenantDoesNotExist(Tenants tenants, TenantName tenant) { + if (tenants.tenantsCopy().containsKey(tenant)) { + throw new BadRequestException("There already exists a tenant '" + tenant + "'"); + } + } + + public static TenantName getTenantFromRequest(HttpRequest request) { + BindingMatch<?> bm = getBindingMatch(request, "http://*/application/v2/tenant/*"); + return TenantName.from(bm.group(2)); + } + + public static TenantName getTenantFromSessionRequest(HttpRequest request) { + BindingMatch<?> bm = getBindingMatch(request, "http://*/application/v2/tenant/*/session*"); + return TenantName.from(bm.group(2)); + } + + public static TenantName getTenantFromApplicationsRequest(HttpRequest request) { + BindingMatch<?> bm = getBindingMatch(request, "http://*/application/v2/tenant/*/application*"); + return TenantName.from(bm.group(2)); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java new file mode 100644 index 00000000000..ba7eff7c461 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.ContentRequest; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.session.LocalSession; + +/** + * Represents a content request for an application. + * + * @author lulf + * @since 5.3 + */ +class ApplicationContentRequest extends ContentRequest { + + private static final String uriPattern = "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*"; + private final ApplicationId applicationId; + private final Zone zone; + + private ApplicationContentRequest(HttpRequest request, LocalSession session, ApplicationId applicationId, Zone zone) { + super(request, session); + this.applicationId = applicationId; + this.zone = zone; + } + + static ContentRequest create(HttpRequest request, LocalSession session, ApplicationId applicationId, Zone zone) { + return new ApplicationContentRequest(request, session, applicationId, zone); + } + + @Override + protected String getContentPath(HttpRequest request) { + BindingMatch<?> bm = Utils.getBindingMatch(request, uriPattern); + return bm.group(7); + } + + @Override + public String getPathPrefix() { + StringBuilder sb = new StringBuilder(); + sb.append("/application/v2/tenant/").append(applicationId.tenant().value()); + sb.append("/application/").append(applicationId.application().value()); + sb.append("/environment/").append(zone.environment().value()); + sb.append("/region/").append(zone.region().value()); + sb.append("/instance/").append(applicationId.instance().value()); + return sb.toString(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java new file mode 100644 index 00000000000..458a3899d4c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.*; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.RotationsCache; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; +import com.yahoo.vespa.config.server.http.*; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.config.server.session.RemoteSessionRepo; + +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * Handler for deleting a currently active application for a tenant. + * + * @author musum + * @since 5.4 + */ +public class ApplicationHandler extends HttpHandler { + + private static final String REQUEST_PROPERTY_TIMEOUT = "timeout"; + private final Tenants tenants; + private final ContentHandler contentHandler = new ContentHandler(); + private final Optional<Provisioner> hostProvisioner; + private final ApplicationConvergenceChecker convergeChecker; + private final Zone zone; + private final LogServerLogGrabber logServerLogGrabber; + + public ApplicationHandler(Executor executor, AccessLog accessLog, Tenants tenants, + HostProvisionerProvider hostProvisionerProvider, Zone zone, + ApplicationConvergenceChecker convergeChecker, + LogServerLogGrabber logServerLogGrabber) { + super(executor, accessLog); + this.tenants = tenants; + this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); + this.zone = zone; + this.convergeChecker = convergeChecker; + this.logServerLogGrabber = logServerLogGrabber; + } + + @Override + public HttpResponse handleDELETE(HttpRequest request) { + ApplicationId applicationId = getApplicationIdFromRequest(request); + Tenant tenant = verifyTenantAndApplication(applicationId); + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + final long sessionId = applicationRepo.getSessionIdForApplication(applicationId); + final LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); + final LocalSession session = localSessionRepo.getSession(sessionId); + if (session == null) { + return HttpErrorResponse.notFoundError("Unable to delete " + applicationId + " (session id " + sessionId + "):" + + "No local deployment for this application found on this config server"); + } + log.log(LogLevel.INFO, "Deleting " + applicationId); + localSessionRepo.removeSession(session.getSessionId()); + session.delete(); + RotationsCache rotationsCache = new RotationsCache(tenant.getCurator(), tenant.getPath()); + rotationsCache.deleteRotationFromZooKeeper(applicationId); + applicationRepo.deleteApplication(applicationId); + if (hostProvisioner.isPresent()) { + hostProvisioner.get().removed(applicationId); + } + return new DeleteApplicationResponse(Response.Status.OK, applicationId); + } + + + @Override + public HttpResponse handleGET(HttpRequest request) { + ApplicationId applicationId = getApplicationIdFromRequest(request); + Tenant tenant = verifyTenantAndApplication(applicationId); + + if (isServiceConvergeRequest(request)) { + Application application = getApplication(tenant, applicationId); + return convergeChecker.nodeConvergenceCheck(application, getHostFromRequest(request), request.getUri()); + } + if (isContentRequest(request)) { + LocalSession session = SessionHandler.getSessionFromRequest(tenant.getLocalSessionRepo(), tenant.getApplicationRepo().getSessionIdForApplication(applicationId)); + return contentHandler.get(ApplicationContentRequest.create(request, session, applicationId, zone)); + } + Application application = getApplication(tenant, applicationId); + + // TODO: Remove this once the config convegence logic is moved to client and is live for all clusters. + if (isConvergeRequest(request)) { + try { + convergeChecker.waitForConfigConverged(application, new TimeoutBudget(Clock.systemUTC(), durationFromRequestTimeout(request))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + if (isServiceConvergeListRequest(request)) { + return convergeChecker.listConfigConvergence(application, request.getUri()); + } + return new GetApplicationResponse(Response.Status.OK, application.getApplicationGeneration()); + } + + @Override + public HttpResponse handlePOST(HttpRequest request) { + ApplicationId applicationId = getApplicationIdFromRequest(request); + Tenant tenant = verifyTenantAndApplication(applicationId); + if (request.getUri().getPath().endsWith("restart")) + return handlePostRestart(request, applicationId); + if (request.getUri().getPath().endsWith("log")) + return handlePostLog(request, applicationId, tenant); + throw new NotFoundException("Illegal POST request '" + request.getUri() + "': Must end by /restart or /log"); + } + + private HttpResponse handlePostRestart(HttpRequest request, ApplicationId applicationId) { + if (getBindingMatch(request).groupCount() != 7) + throw new NotFoundException("Illegal POST restart request '" + request.getUri() + + "': Must have 6 arguments but had " + ( getBindingMatch(request).groupCount()-1 ) ); + if (hostProvisioner.isPresent()) + hostProvisioner.get().restart(applicationId, hostFilterFrom(request)); + return new JSONResponse(Response.Status.OK); // return empty + } + + private HttpResponse handlePostLog(HttpRequest request, ApplicationId applicationId, Tenant tenant) { + if (getBindingMatch(request).groupCount() != 7) + throw new NotFoundException("Illegal POST log request '" + request.getUri() + + "': Must have 6 arguments but had " + ( getBindingMatch(request).groupCount()-1 ) ); + Application application = getApplication(tenant, applicationId); + return logServerLogGrabber.grabLog(application); + } + + private HostFilter hostFilterFrom(HttpRequest request) { + return HostFilter.from(request.getProperty("hostname"), + request.getProperty("flavor"), + request.getProperty("clusterType"), + request.getProperty("clusterId")); + } + + private Tenant verifyTenantAndApplication(ApplicationId applicationId) { + Tenant tenant = Utils.checkThatTenantExists(tenants, applicationId.tenant()); + List<ApplicationId> applicationIds = listApplicationIds(tenant); + if ( ! applicationIds.contains(applicationId)) { + throw new NotFoundException("No such application id: " + applicationId); + } + return tenant; + } + + private Duration durationFromRequestTimeout(HttpRequest request) { + long timeoutInSeconds = 60; + if (request.hasProperty(REQUEST_PROPERTY_TIMEOUT)) { + timeoutInSeconds = Long.parseLong(request.getProperty(REQUEST_PROPERTY_TIMEOUT)); + } + return Duration.ofSeconds(timeoutInSeconds); + } + + private Application getApplication(Tenant tenant, ApplicationId applicationId) { + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + RemoteSessionRepo remoteSessionRepo = tenant.getRemoteSessionRepo(); + long sessionId = applicationRepo.getSessionIdForApplication(applicationId); + RemoteSession session = remoteSessionRepo.getSession(sessionId, 0); + return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty()); + } + + private List<ApplicationId> listApplicationIds(Tenant tenant) { + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + return applicationRepo.listApplications(); + } + + // Note: Update src/main/resources/configserver-app/services.xml if you do any changes to the bindings + private static BindingMatch<?> getBindingMatch(HttpRequest request) { + return HttpConfigRequests.getBindingMatch(request, + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*", + "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*", + "http://*/application/v2/tenant/*/application/*"); + } + + private static boolean isConvergeRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 7 && + request.getUri().getPath().endsWith("converge"); + } + + private static boolean isServiceConvergeListRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 7 && + request.getUri().getPath().endsWith("serviceconverge"); + } + + private static boolean isServiceConvergeRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() == 8 && + request.getUri().getPath().contains("/serviceconverge/"); + } + + + private static boolean isContentRequest(HttpRequest request) { + return getBindingMatch(request).groupCount() > 7; + } + + private static String getHostFromRequest(HttpRequest req) { + BindingMatch<?> bm = getBindingMatch(req); + return bm.group(7); + } + + private static ApplicationId getApplicationIdFromRequest(HttpRequest req) { + // Two bindings for this: with full app id or only application name + BindingMatch<?> bm = getBindingMatch(req); + if (bm.groupCount() > 4) return createFromRequestFullAppId(bm); + return createFromRequestSimpleAppId(bm); + } + + // The URL pattern with only tenant and application given + private static ApplicationId createFromRequestSimpleAppId(BindingMatch<?> bm) { + TenantName tenant = TenantName.from(bm.group(2)); + ApplicationName application = ApplicationName.from(bm.group(3)); + return new ApplicationId.Builder().tenant(tenant).applicationName(application).build(); + } + + // The URL pattern with full app id given + private static ApplicationId createFromRequestFullAppId(BindingMatch<?> bm) { + String tenant = bm.group(2); + String application = bm.group(3); + String instance = bm.group(6); + return new ApplicationId.Builder() + .tenant(tenant) + .applicationName(application).instanceName(instance) + .build(); + } + + private static class DeleteApplicationResponse extends JSONResponse { + public DeleteApplicationResponse(int status, ApplicationId applicationId) { + super(status); + object.setString("message", "Application '" + applicationId + "' deleted"); + } + } + + private static class GetApplicationResponse extends JSONResponse { + public GetApplicationResponse(int status, long generation) { + super(status); + object.setLong("generation", generation); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java new file mode 100644 index 00000000000..086954c384f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.HostRegistries; +import com.yahoo.vespa.config.server.HostRegistry; +import com.yahoo.vespa.config.server.http.*; + +import java.util.concurrent.Executor; + + +/** + * Handler for getting tenant and application for a given hostname. + * + * @author musum + * @since 5.19 + */ +public class HostHandler extends HttpHandler { + final HostRegistries hostRegistries; + private final Zone zone; + + public HostHandler(Executor executor, AccessLog accessLog, GlobalComponentRegistry globalComponentRegistry) { + super(executor, accessLog); + this.hostRegistries = globalComponentRegistry.getHostRegistries(); + this.zone = globalComponentRegistry.getZone(); + } + + @Override + public HttpResponse handleGET(HttpRequest request) { + String hostname = getBindingMatch(request).group(2); + log.log(LogLevel.DEBUG, "hostname=" + hostname); + + HostRegistry<TenantName> tenantHostRegistry = hostRegistries.getTenantHostRegistry(); + log.log(LogLevel.DEBUG, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); + TenantName tenant = tenantHostRegistry.getKeyForHost(hostname); + if (tenant == null) return createError(hostname); + log.log(LogLevel.DEBUG, "tenant=" + tenant); + HostRegistry<ApplicationId> applicationIdHostRegistry = hostRegistries.getApplicationHostRegistry(tenant); + ApplicationId applicationId; + if (applicationIdHostRegistry == null) return createError(hostname); + applicationId = applicationIdHostRegistry.getKeyForHost(hostname); + log.log(LogLevel.DEBUG, "applicationId=" + applicationId); + if (applicationId == null) { + return createError(hostname); + } else { + log.log(LogLevel.DEBUG, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); + return new HostResponse(Response.Status.OK, applicationId, zone); + } + } + + private HttpErrorResponse createError(String hostname) { + return HttpErrorResponse.notFoundError("Could not find any application using host '" + hostname + "'"); + } + + private static BindingMatch<?> getBindingMatch(HttpRequest request) { + return HttpConfigRequests.getBindingMatch(request, "http://*/application/v2/host/*"); + } + + private static class HostResponse extends JSONResponse { + public HostResponse(int status, ApplicationId applicationId, Zone zone) { + super(status); + object.setString("tenant", applicationId.tenant().value()); + object.setString("application", applicationId.application().value()); + object.setString("environment", zone.environment().value()); + object.setString("region", zone.region().value()); + object.setString("instance", applicationId.instance().value()); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java new file mode 100644 index 00000000000..b63c52a26bb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpConfigRequests.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.net.URI; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.jdisc.application.UriPattern; +import com.yahoo.jdisc.application.UriPattern.Match; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.http.NotFoundException; + +/** + * Helpers for v2 config REST API + * + * @author vegardh + */ +public class HttpConfigRequests { + + static final String RECURSIVE_QUERY_PROPERTY = "recursive"; + + /** + * Produces the binding match for the request. If it's not available on the jDisc request, create one for + * testing using the long and short app id URL patterns given. + * @param req an {@link com.yahoo.container.jdisc.HttpRequest} + * @param patterns A list of patterns that should be matched if no match on binding. + * @return match + */ + public static BindingMatch<?> getBindingMatch(HttpRequest req, String ... patterns) { + com.yahoo.jdisc.http.HttpRequest jDiscRequest = req.getJDiscRequest(); + if (jDiscRequest==null) throw new IllegalArgumentException("No JDisc request for: " + req.getUri()); + BindingMatch<?> jdBm = jDiscRequest.getBindingMatch(); + if (jdBm!=null) return jdBm; + + // If not, use provided patterns + for (String pattern : patterns) { + UriPattern fullAppIdPattern = new UriPattern(pattern); + URI uri = req.getUri(); + Match match = fullAppIdPattern.match(uri); + if (match!=null) return new BindingMatch<>(match, new Object()); + } + throw new IllegalArgumentException("Illegal url for config request: " + req.getUri()); + } + + + static RequestHandler getRequestHandler(Tenants tenants, TenantRequest request) { + Tenant tenant = tenants.tenantsCopy().get(request.getApplicationId().tenant()); + if (tenant==null) throw new NotFoundException("No such tenant: "+request.getApplicationId().tenant()); + return tenant.getRequestHandler(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java new file mode 100644 index 00000000000..6c872578324 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.inject.Inject; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.http.HttpConfigRequest; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.HttpHandler; + +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * HTTP handler for a getConfig operation + * + * @author lulf + * @since 5.1 + */ +public class HttpGetConfigHandler extends HttpHandler { + + private final Tenants tenants; + + @Inject + public HttpGetConfigHandler(Executor executor, AccessLog accesslog, Tenants tenants) { + super(executor, accesslog); + this.tenants = tenants; + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + HttpConfigRequest request = HttpConfigRequest.createFromRequestV2(req); + RequestHandler requestHandler = HttpConfigRequests.getRequestHandler(tenants, request); + HttpConfigRequest.validateRequestKey(request.getConfigKey(), requestHandler, request.getApplicationId()); + return HttpConfigResponse.createFromConfig(resolveConfig(request, requestHandler)); + } + + private ConfigResponse resolveConfig(HttpConfigRequest request, RequestHandler requestHandler) { + log.log(LogLevel.DEBUG, "nocache=" + request.noCache()); + ConfigResponse config = requestHandler.resolveConfig(request.getApplicationId(), request, Optional.empty()); + if (config == null) HttpConfigRequest.throwModelNotReady(); + return config; + } +}
\ No newline at end of file diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java new file mode 100644 index 00000000000..0358a9a7046 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandler.java @@ -0,0 +1,180 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.*; +import java.util.concurrent.Executor; +import com.google.inject.Inject; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.path.Path; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.Utils; +import static com.yahoo.jdisc.http.HttpResponse.Status.*; + +/** + * Handler for a list configs operation. Lists all configs in model for a given application and tenant. + * + * @author vegardh + * @since 5.3 + */ +public class HttpListConfigsHandler extends HttpHandler { + private final Tenants tenants; + private final Zone zone; + + @Inject + public HttpListConfigsHandler(Executor executor, AccessLog accesslog, Tenants tenants, Zone zone) { + super(executor, accesslog); + this.tenants = tenants; + this.zone = zone; + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + HttpListConfigsRequest listReq = HttpListConfigsRequest.createFromListRequest(req); + RequestHandler requestHandler = HttpConfigRequests.getRequestHandler(tenants, listReq); + ApplicationId appId = listReq.getApplicationId(); + Set<ConfigKey<?>> configs = requestHandler.listConfigs(appId, Optional.empty(), listReq.isRecursive()); + String urlBase = getUrlBase(req, listReq, appId, zone); + Set<ConfigKey<?>> allConfigs = requestHandler.allConfigsProduced(appId, Optional.empty()); + return new ListConfigsResponse(configs, allConfigs, urlBase, listReq.isRecursive()); + } + + static String getUrlBase(HttpRequest req, HttpListConfigsRequest listReq, ApplicationId appId, Zone zone) { + if (listReq.isFullAppId()) + return Utils.getUrlBase(req, + Path.fromString("/config/v2/tenant/"). + append(appId.tenant().value()). + append("application"). + append(appId.application().value()). + append("environment"). + append(zone.environment().value()). + append("region"). + append(zone.region().value()). + append("instance"). + append(appId.instance().value()).getAbsolute()+"/"); + else + return Utils.getUrlBase(req, + Path.fromString("/config/v2/tenant/"). + append(appId.tenant().value()). + append("application"). + append(appId.application().value()).getAbsolute()+"/"); + } + + static class ListConfigsResponse extends HttpResponse { + private final List<ConfigKey<?>> configs; + private final Set<ConfigKey<?>> allConfigs; + private final String urlBase; + private final boolean recursive; + + /** + * New list response + * + * @param configs the configs to include in the list + * @param urlBase for example "http://foo.com:19071/config/v1/ (configs are appended to the listed URLs based on configs list) + * @param recursive list recursively + */ + public ListConfigsResponse(Set<ConfigKey<?>> configs, Set<ConfigKey<?>> allConfigs, String urlBase, boolean recursive) { + super(OK); + this.configs = new ArrayList<>(configs); + Collections.sort(this.configs); + this.allConfigs = allConfigs; + this.urlBase = urlBase; + this.recursive = recursive; + } + + /** + * The listing URL for this config in this service + * + * @param key config key + * @param rec recursive + * @return url + */ + public String toUrl(ConfigKey<?> key, boolean rec) { + return urlBase + key.getNamespace() + "." + key.getName() + configIdUrlPart(rec, key.getConfigId()); + } + + // Do not end with / if it's a recursive listing. Furthermore, don't do it if it's the empty config id (special handling of the root config id). + private String configIdUrlPart(boolean rec, String configId) { + if ("".equals(configId)) return ""; + if (rec) return "/" + configId; + return "/" + configId + "/"; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor array; + if (!recursive) { + array = root.setArray("children"); + for (ConfigKey<?> key : keysThatHaveAChildWithSameName(configs, allConfigs)) { + array.addString(toUrl(key, false)); + } + } + array = root.setArray("configs"); + + Collection<ConfigKey<?>> cfs; + if (recursive) { + cfs = configs; + } else { + cfs = withParentConfigId(configs); + } + for (ConfigKey<?> key : cfs) { + array.addString(toUrl(key, true)); + } + new JsonFormat(true).encode(outputStream, slime); + } + + public static Set<ConfigKey<?>> keysThatHaveAChildWithSameName(Collection<ConfigKey<?>> keys, Set<ConfigKey<?>> allConfigs) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> k : keys) { + if (ListConfigsResponse.hasAChild(k, allConfigs)) ret.add(k); + } + return ret; + } + + // Do we do this already somewhere? + private static Set<ConfigKey<?>> withParentConfigId(Collection<ConfigKey<?>> keys) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> k : keys) { + ret.add(new ConfigKey<>(k.getName(), parentConfigId(k.getConfigId()), k.getNamespace())); + } + return ret; + } + + static String parentConfigId(String id) { + if (id==null) return null; + if (!id.contains("/")) return ""; + return id.substring(0, id.lastIndexOf('/')); + } + + static boolean hasAChild(ConfigKey<?> key, Set<ConfigKey<?>> keys) { + if ("".equals(key.getConfigId())) return false; + for (ConfigKey<?> k : keys) { + if (!k.getName().equals(key.getName())) continue; + if ("".equals(k.getConfigId())) continue; + if (k.getConfigId().equals(key.getConfigId())) continue; + if (k.getConfigId().startsWith(key.getConfigId())) return true; + } + return false; + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java new file mode 100644 index 00000000000..1140d6e769f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsRequest.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.HttpConfigRequest; + +/** + * A request to list config, bound to tenant and app id. Optionally bound to a config key, for request for named config. + * + * @author vegardh + * + */ +public class HttpListConfigsRequest implements TenantRequest { + private final ConfigKey<?> key; // non-null if it's a named list request + private final ApplicationId appId; + private final boolean recursive; + private final boolean fullAppId; + + private HttpListConfigsRequest(ConfigKey<?> key, ApplicationId appId, boolean recursive, boolean fullAppId) { + this.key = key; + this.appId = appId; + this.recursive = recursive; + this.fullAppId = fullAppId; + } + + public static HttpListConfigsRequest createFromListRequest(HttpRequest req) { + BindingMatch<?> bm = HttpConfigRequests.getBindingMatch(req, + "http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/", + "http://*/config/v2/tenant/*/application/*/"); + if (bm.groupCount()>4) return createFromListRequestFullAppId(req, bm); + return createFromListRequestSimpleAppId(req, bm); + } + + private static HttpListConfigsRequest createFromListRequestSimpleAppId(HttpRequest req, BindingMatch<?> bm) { + TenantName tenant = TenantName.from(bm.group(2)); + ApplicationName application = ApplicationName.from(bm.group(3)); + return new HttpListConfigsRequest(null, new ApplicationId.Builder().tenant(tenant).applicationName(application).build(), + req.getBooleanProperty(HttpConfigRequests.RECURSIVE_QUERY_PROPERTY), false); + } + + private static HttpListConfigsRequest createFromListRequestFullAppId(HttpRequest req, BindingMatch<?> bm) { + String tenant = bm.group(2); + String application = bm.group(3); + String environment = bm.group(4); + String region = bm.group(5); + String instance = bm.group(6); + + ApplicationId appId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName(application) + .instanceName(instance) + .build(); + return new HttpListConfigsRequest(null, appId, + req.getBooleanProperty(HttpConfigRequests.RECURSIVE_QUERY_PROPERTY), true); + } + + public static HttpListConfigsRequest createFromNamedListRequest(HttpRequest req) { + // http://*/config/v2/tenant/*/application/*/*/ + // http://*/config/v2/tenant/*/application/*/*/*/ + // http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/ + // http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/ + BindingMatch<?> bm = HttpConfigRequests.getBindingMatch(req, + "http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/", + "http://*/config/v2/tenant/*/application/*/*/*/"); + if (bm.groupCount()>6) return createFromNamedListRequestFullAppId(req, bm); + return createFromNamedListRequestSimpleAppId(req, bm); + } + + private static HttpListConfigsRequest createFromNamedListRequestSimpleAppId(HttpRequest req, BindingMatch<?> bm) { + TenantName tenant = TenantName.from(bm.group(2)); + ApplicationName application = ApplicationName.from(bm.group(3)); + String conf = bm.group(4); + String cId; + String cName; + String cNamespace; + if (bm.groupCount() >= 6) { + cId = bm.group(5); + } else { + cId = ""; + } + Tuple2<String, String> nns = HttpConfigRequest.nameAndNamespace(conf); + cName = nns.first; + cNamespace = nns.second; + ConfigKey<?> key = new ConfigKey<>(cName, cId, cNamespace); + return new HttpListConfigsRequest(key, new ApplicationId.Builder().tenant(tenant).applicationName(application).build(), + req.getBooleanProperty(HttpConfigRequests.RECURSIVE_QUERY_PROPERTY), false); + } + + private static HttpListConfigsRequest createFromNamedListRequestFullAppId(HttpRequest req, BindingMatch<?> bm) { + String tenant = bm.group(2); + String application = bm.group(3); + String environment = bm.group(4); + String region = bm.group(5); + String instance = bm.group(6); + String conf = bm.group(7); + String cId; + String cName; + String cNamespace; + if (bm.groupCount() >= 9) { + cId = bm.group(8); + } else { + cId = ""; + } + Tuple2<String, String> nns = HttpConfigRequest.nameAndNamespace(conf); + cName = nns.first; + cNamespace = nns.second; + ConfigKey<?> key = new ConfigKey<>(cName, cId, cNamespace); + ApplicationId appId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName(application) + .instanceName(instance) + .build(); + return new HttpListConfigsRequest(key, appId, + req.getBooleanProperty(HttpConfigRequests.RECURSIVE_QUERY_PROPERTY), true); + } + + /** + * The application id of the request + * @return app id + */ + @Override + public ApplicationId getApplicationId() { + return appId; + } + + /** + * True if the request was of the recursive form + * @return recursive + */ + public boolean isRecursive() { + return recursive; + } + + /** + * True if this was created using a URL with tenant, application, environment, region and instance; false if only tenant and application + * @return true if full app id used + */ + public boolean isFullAppId() { + return fullAppId; + } + + /** + * Returns the config key of the request if it was for a named config, or null if it was just a listing request + * @return key or null + */ + public ConfigKey<?> getKey() { + return key; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java new file mode 100644 index 00000000000..ed4accb386b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpListNamedConfigsHandler.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executor; + +import com.google.inject.Inject; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.HttpConfigRequest; +import com.yahoo.vespa.config.server.http.HttpHandler; + +/** + * Handler for a list named configs operation. Lists all configs in model for a given application and tenant, config name and optionally config id. + * + * @author vegardh + * @since 5.3 + */ +public class HttpListNamedConfigsHandler extends HttpHandler { + + private final Tenants tenants; + private final Zone zone; + + @Inject + public HttpListNamedConfigsHandler(Executor executor, AccessLog accesslog, Tenants tenants, Zone zone) { + super(executor, accesslog); + this.tenants = tenants; + this.zone = zone; + } + + @Override + public HttpResponse handleGET(HttpRequest req) { + HttpListConfigsRequest listReq = HttpListConfigsRequest.createFromNamedListRequest(req); + RequestHandler requestHandler = HttpConfigRequests.getRequestHandler(tenants, listReq); + ApplicationId appId = listReq.getApplicationId(); + HttpConfigRequest.validateRequestKey(listReq.getKey(), requestHandler, appId); + Set<ConfigKey<?>> configs = requestHandler.listNamedConfigs(appId, Optional.empty(), listReq.getKey(), listReq.isRecursive()); + String urlBase = HttpListConfigsHandler.getUrlBase(req, listReq, appId, zone); + Set<ConfigKey<?>> allConfigs = requestHandler.allConfigsProduced(appId, Optional.empty()); + return new HttpListConfigsHandler.ListConfigsResponse(configs, allConfigs, urlBase, listReq.isRecursive()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java new file mode 100644 index 00000000000..e77c3928f11 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.Utils; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; + +/** + * Handler for listing currently active applications for a tenant. + * + * @author lulf + * @since 5.1 + */ +public class ListApplicationsHandler extends HttpHandler { + private final Tenants tenants; + private final Zone zone; + public ListApplicationsHandler(Executor executor, AccessLog accessLog, Tenants tenants, Zone zone) { + super(executor, accessLog); + this.tenants = tenants; + this.zone = zone; + } + + @Override + public HttpResponse handleGET(HttpRequest request) { + TenantName tenantName = Utils.getTenantFromApplicationsRequest(request); + final String urlBase = Utils.getUrlBase(request, "/application/v2/tenant/" + tenantName + "/application/"); + + List<ApplicationId> applicationIds = listApplicationIds(tenantName); + Collection<String> applicationUrls = Collections2.transform(applicationIds, new Function<ApplicationId, String>() { + @Override + public String apply(ApplicationId id) { + return createUrlStringFromId(urlBase, id, zone); + } + }); + return new ListApplicationsResponse(Response.Status.OK, applicationUrls); + } + + private List<ApplicationId> listApplicationIds(TenantName tenantName) { + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + return applicationRepo.listApplications(); + } + + private static String createUrlStringFromId(String urlBase, ApplicationId id, Zone zone) { + StringBuilder sb = new StringBuilder(); + sb.append(urlBase).append(id.application().value()); + sb.append("/environment/").append(zone.environment().value()); + sb.append("/region/").append(zone.region().value()); + sb.append("/instance/").append(id.instance().value()); + return sb.toString(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java new file mode 100644 index 00000000000..3efd50fade6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsResponse.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collection; + +/** + * Response that lists applications. + * + * @author lulf + * @since 5.1 + */ +public class ListApplicationsResponse extends HttpResponse { + private final Slime slime = new Slime(); + public ListApplicationsResponse(int status, Collection<String> applications) { + super(status); + Cursor array = slime.setArray(); + for (String url : applications) { + array.addString(url); + } + } + + @Override + public void render(OutputStream outputStream) throws IOException { + new JsonFormat(true).encode(outputStream, slime); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java new file mode 100644 index 00000000000..0fd456b6bd0 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsHandler.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.util.concurrent.Executor; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.Tenants; + +/** + * Handler to list tenants in the configserver + * + * @author vegardh + * + */ +public class ListTenantsHandler extends HttpHandler { + + private final Tenants tenants; + + + public ListTenantsHandler(Executor executor, AccessLog accessLog, Tenants tenants) { + super(executor, accessLog); + this.tenants = tenants; + } + + + @Override + protected HttpResponse handleGET(HttpRequest request) { + return new ListTenantsResponse(tenants.tenantsCopy().keySet()); + } + + + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java new file mode 100644 index 00000000000..35ff6faa89f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + + +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Cursor; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.SessionResponse; + +import java.util.Collection; + +/** + * Tenant list response + * + * @author vegardh + * + */ +public class ListTenantsResponse extends SessionResponse { + private final Collection<TenantName> tenantNames; + + public ListTenantsResponse(final Collection<TenantName> tenants) { + super(); + this.tenantNames = tenants; + Cursor tenantArray = this.root.setArray("tenants"); + synchronized (tenants) { + for (final TenantName tenantName : tenants) { + tenantArray.addString(tenantName.value()); + } + } + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + + public Collection<TenantName> getTenantNames() { + return tenantNames; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java new file mode 100644 index 00000000000..ee926f39cad --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.util.Optional; +import java.util.concurrent.Executor; + +import com.google.inject.Inject; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.http.SessionActiveHandlerBase; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSession; + +/** + * Handler that activates a session given by tenant and id (PUT). + * + * @author vegardh + * @since 5.1 + */ +public class SessionActiveHandler extends SessionActiveHandlerBase { + + private final Tenants tenants; + private final Optional<Provisioner> hostProvisioner; + private final Zone zone; + + @Inject + public SessionActiveHandler(Executor executor, + AccessLog accessLog, + Tenants tenants, + HostProvisionerProvider hostProvisionerProvider, + Zone zone) { + super(executor, accessLog); + this.tenants = tenants; + this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); + this.zone = zone; + } + + @Override + protected HttpResponse handlePUT(HttpRequest request) { + TimeoutBudget timeoutBudget = getTimeoutBudget(request, SessionHandler.DEFAULT_ACTIVATE_TIMEOUT); + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + LocalSession localSession = getSessionFromRequestV2(tenant.getLocalSessionRepo(), request); + activate(request, tenant.getLocalSessionRepo(), tenant.getActivateLock(), timeoutBudget, hostProvisioner, localSession); + return new SessionActiveResponse(localSession.getMetaData().getSlime(), tenantName, request, localSession, zone); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java new file mode 100644 index 00000000000..b5948c7e786 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.slime.Slime; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.vespa.config.server.session.LocalSession; + +public class SessionActiveResponse extends SessionResponse { + + public SessionActiveResponse(Slime metaData, TenantName tenantName, HttpRequest request, LocalSession session, Zone zone) { + super(metaData, metaData.get()); + String message = "Session " + session.getSessionId() + " for tenant '" + tenantName + "' activated."; + root.setString("tenant", tenantName.value()); + root.setString("message", message); + final ApplicationId applicationId = session.getApplicationId(); + root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + + "/application/v2/tenant/" + tenantName + + "/application/" + applicationId.application().value() + + "/environment/" + zone.environment().value() + + "/region/" + zone.region().value() + + "/instance/" + applicationId.instance().value()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java new file mode 100644 index 00000000000..669ec049770 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandler.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.inject.Inject; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.http.ContentHandler; +import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; + +import java.util.concurrent.Executor; + +/** + * A handler that will return content or content status for files or directories + * in the session's application package + * + * @author lulf + * @since 5.1 + */ +public class SessionContentHandler extends SessionHandler { + private final Tenants tenants; + private final ContentHandler contentHandler = new ContentHandler(); + + @Inject + public SessionContentHandler(Executor executor, AccessLog accessLog, Tenants tenants) { + super(executor, accessLog); + this.tenants = tenants; + } + + @Override + public HttpResponse handleGET(HttpRequest request) { + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + LocalSession session = getLocalSession(request, tenant.getLocalSessionRepo()); + return contentHandler.get(SessionContentRequestV2.create(request, session)); + } + + private LocalSession getLocalSession(HttpRequest request, LocalSessionRepo localSessionRepo) { + LocalSession session = getSessionFromRequestV2(localSessionRepo, request); + if (session == null) { + throw new NotFoundException("No valid session id in request " + request.getUri().toString()); + } + return session; + } + + @Override + public HttpResponse handlePUT(HttpRequest request) { + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + return contentHandler.put(SessionContentRequestV2.create(request, getSessionFromRequestV2(tenant.getLocalSessionRepo(), request))); + } + + @Override + public HttpResponse handleDELETE(HttpRequest request) { + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + return contentHandler.delete(SessionContentRequestV2.create(request, getSessionFromRequestV2(tenant.getLocalSessionRepo(), request))); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java new file mode 100644 index 00000000000..16d3fd6802a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionContentRequestV2.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.vespa.config.server.http.ContentRequest; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.session.LocalSession; + +/** + * Requests for content and content status (v2) + * are handled by this class. + * + * @author musum + * @since 5.3 + */ +class SessionContentRequestV2 extends ContentRequest { + private static final String uriPattern = "http://*/application/v2/tenant/*/session/*/content/*"; + private final TenantName tenantName; + private final long sessionId; + + private SessionContentRequestV2(HttpRequest request, LocalSession session, TenantName tenantName) { + super(request, session); + this.tenantName = tenantName; + this.sessionId = session.getSessionId(); + } + + static ContentRequest create(HttpRequest request, LocalSession session) { + return new SessionContentRequestV2(request, session, Utils.getTenantFromSessionRequest(request)); + } + + @Override + public String getPathPrefix() { + return "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; + } + + @Override + protected String getContentPath(HttpRequest request) { + BindingMatch<?> bm = Utils.getBindingMatch(request, uriPattern); + return bm.group(4); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java new file mode 100644 index 00000000000..90d8ba63892 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.application.UriPattern; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.SessionCreate; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; + +import java.net.URI; +import java.time.Duration; +import java.util.concurrent.Executor; +import java.util.logging.Logger; + +/** + * A handler that is able to create a session from an application package, + * or create a new session from a previous session (with id or the "active" session). + * Handles /application/v2/ requests + * + * @author musum + * @since 5.1 + */ +public class SessionCreateHandler extends SessionHandler { + private static final Logger log = Logger.getLogger(SessionCreateHandler.class.getName()); + private final Tenants tenants; + private static final String fromPattern = "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*"; + private final ConfigserverConfig configserverConfig; + + @Inject + public SessionCreateHandler(Executor executor, AccessLog accessLog, Tenants tenants, ConfigserverConfig configserverConfig) { + super(executor, accessLog); + this.tenants = tenants; + this.configserverConfig = configserverConfig; + } + + @Override + protected HttpResponse handlePOST(HttpRequest request) { + Slime deployLog = createDeployLog(); + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + final SessionCreate sessionCreate = new SessionCreate(tenant.getSessionFactory(), tenant.getLocalSessionRepo(), + new SessionCreateResponseV2(tenant, deployLog, deployLog.get())); + TimeoutBudget timeoutBudget = SessionHandler.getTimeoutBudget(request, Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout())); + if (request.hasProperty("from")) { + LocalSession fromSession = getExistingSession(tenant, request); + return sessionCreate.createFromExisting(request, deployLog, fromSession, tenantName, timeoutBudget); + } else { + return sessionCreate.create(request, deployLog, tenantName, timeoutBudget); + } + } + + private static LocalSession getExistingSession(Tenant tenant, HttpRequest request) { + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); + ApplicationId applicationId = getFromProperty(request); + return SessionHandler.getSessionFromRequest(localSessionRepo, applicationRepo.getSessionIdForApplication(applicationId)); + } + + private static ApplicationId getFromProperty(HttpRequest request) { + String from = request.getProperty("from"); + if (from == null || "".equals(from)) { + throw new BadRequestException("Parameter 'from' has illegal value '" + from + "'"); + } + return getAndValidateFromParameter(URI.create(from)); + } + + private static ApplicationId getAndValidateFromParameter(URI from) { + UriPattern.Match match = new UriPattern(fromPattern).match(from); + if (match == null || match.groupCount() < 7) { + throw new BadRequestException("Parameter 'from' has illegal value '" + from + "'"); + } + return new ApplicationId.Builder() + .tenant(match.group(2)) + .applicationName(match.group(3)) + .instanceName(match.group(6)).build(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponseV2.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponseV2.java new file mode 100644 index 00000000000..5fab4b97407 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponseV2.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.http.SessionCreateResponse; +import com.yahoo.vespa.config.server.http.SessionResponse; + +/** + * Creates a response for SessionCreateHandler (v2). + * + * @author musum + * @since 5.1.27 + */ +public class SessionCreateResponseV2 extends SessionResponse implements SessionCreateResponse { + private final Tenant tenant; + + public SessionCreateResponseV2(Tenant tenant, Slime deployLog, Cursor root) { + super(deployLog, root); + this.tenant = tenant; + } + + @Override + public HttpResponse createResponse(String hostName, int port, long sessionId) { + String tenantName = tenant.getName().value(); + String path = "http://" + hostName + ":" + port + "/application/v2/tenant/" + tenantName + "/session/" + sessionId; + + this.root.setString("tenant", tenantName); + this.root.setString("session-id", Long.toString(sessionId)); + this.root.setString("prepared", path + "/prepared"); + this.root.setString("content", path + "/content/"); + this.root.setString("message", "Session " + sessionId + " for tenant '" + tenantName + "' created."); + return this; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java new file mode 100644 index 00000000000..9a0cc7e6d16 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.configchange.RefeedActions; +import com.yahoo.vespa.config.server.configchange.RefeedActionsFormatter; +import com.yahoo.vespa.config.server.configchange.RestartActionsFormatter; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.session.*; + +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A handler that prepares a session given by an id in the request. v2 of application API + * + * @author musum + * @since 5.1.29 + */ +public class SessionPrepareHandler extends SessionHandler { + private static final Logger log = Logger.getLogger(SessionPrepareHandler.class.getName()); + + private final Tenants tenants; + private final ConfigserverConfig configserverConfig; + + @Inject + public SessionPrepareHandler(Executor executor, + AccessLog accessLog, + Tenants tenants, + ConfigserverConfig configserverConfig) { + super(executor, accessLog); + this.tenants = tenants; + this.configserverConfig = configserverConfig; + } + + @Override + protected HttpResponse handlePUT(HttpRequest request) { + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + Tenant tenantContext = Utils.checkThatTenantExists(tenants, tenantName); + LocalSession session = getSessionFromRequestV2(tenantContext.getLocalSessionRepo(), request); + if (Session.Status.ACTIVATE.equals(session.getStatus())) { + throw new IllegalArgumentException("Session is active: " + session.getSessionId()); + } + log.log(LogLevel.DEBUG, "session=" + session); + boolean verbose = request.getBooleanProperty("verbose"); + Slime rawDeployLog = createDeployLog(); + PrepareParams prepParams = PrepareParams.fromHttpRequest(request, tenantName, configserverConfig); + // An app id currently using only the name + ApplicationId appId = prepParams.getApplicationId(); + DeployLogger logger = createLogger(rawDeployLog, verbose, appId); + ConfigChangeActions actions = session.prepare(logger, prepParams, getCurrentActiveApplicationSet(tenantContext, appId), tenantContext.getPath()); + logConfigChangeActions(actions, logger); + log.log(LogLevel.INFO, Tenants.logPre(appId)+"Session "+session.getSessionId()+" prepared successfully. "); + return new SessionPrepareResponse(rawDeployLog, tenantContext, request, session, actions); + } + + private static void logConfigChangeActions(ConfigChangeActions actions, DeployLogger logger) { + if ( ! actions.getRestartActions().getEntries().isEmpty()) { + logger.log(Level.WARNING, "Change(s) between active and new application that require restart:\n" + + new RestartActionsFormatter(actions.getRestartActions()).format()); + } + if ( ! actions.getRefeedActions().getEntries().isEmpty()) { + boolean allAllowed = actions.getRefeedActions().getEntries().stream().allMatch(RefeedActions.Entry::allowed); + logger.log(allAllowed ? Level.INFO : Level.WARNING, + "Change(s) between active and new application that may require re-feed:\n" + + new RefeedActionsFormatter(actions.getRefeedActions()).format()); + } + } + + @Override + protected HttpResponse handleGET(HttpRequest request) { + TenantName tenantName = Utils.getTenantFromSessionRequest(request); + log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + RemoteSession session = getSessionFromRequestV2(tenant.getRemoteSessionRepo(), request); + if (Session.Status.ACTIVATE.equals(session.getStatus())) + throw new IllegalArgumentException("Session is active: " + session.getSessionId()); + if (!Session.Status.PREPARE.equals(session.getStatus())) + throw new IllegalArgumentException("Session not prepared: " + session.getSessionId()); + return new SessionPrepareResponse(createDeployLog(), tenant, request, session, new ConfigChangeActions()); + } + + private static Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { + Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + try { + long currentActiveSessionId = applicationRepo.getSessionIdForApplication(appId); + final RemoteSession currentActiveSession = tenant.getRemoteSessionRepo().getSession(currentActiveSessionId); + if (currentActiveSession != null) { + currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); + } + } catch (IllegalArgumentException e) { + // Do nothing if we have no currently active session + } + return currentActiveApplicationSet; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java new file mode 100644 index 00000000000..dbc36bbc948 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; +import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.vespa.config.server.session.Session; + +/** + * Creates a response for SessionPrepareHandler. + * + * @author musum + * @since 5.1.28 + */ +class SessionPrepareResponse extends SessionResponse { + + public SessionPrepareResponse(Slime deployLog, Tenant tenant, HttpRequest request, Session session, ConfigChangeActions actions) { + super(deployLog, deployLog.get()); + String message = "Session " + session.getSessionId() + " for tenant '" + tenant.getName() + "' prepared."; + this.root.setString("tenant", tenant.getName().value()); + this.root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenant.getName() + "/session/" + session.getSessionId() + "/active"); + root.setString("message", message); + new ConfigChangeActionsSlimeConverter(actions).toSlime(root); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java new file mode 100644 index 00000000000..469736005b8 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.SessionResponse; + +/** + * Response for tenant create + * + * @author vegardh + * + */ +public class TenantCreateResponse extends SessionResponse { + + public TenantCreateResponse(TenantName tenant) { + super(); + this.root.setString("message", "Tenant "+tenant+" created."); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java new file mode 100644 index 00000000000..dbadc77c9dd --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.SessionResponse; + +/** + * Response for tenant delete + * + * @author vegardh + * + */ +public class TenantDeleteResponse extends SessionResponse { + + public TenantDeleteResponse(TenantName tenant) { + super(); + this.root.setString("message", "Tenant "+tenant+" deleted."); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java new file mode 100644 index 00000000000..99393cd351a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; +import com.yahoo.vespa.config.server.http.SessionResponse; + +/** + * Response for tenant create + * + * @author musum + */ +public class TenantGetResponse extends SessionResponse { + + public TenantGetResponse(TenantName tenant) { + super(); + this.root.setString("message", "Tenant '" + tenant + "' exists."); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java new file mode 100644 index 00000000000..e373eb4478f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.util.List; +import java.util.concurrent.Executor; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.InternalServerException; +import com.yahoo.vespa.config.server.http.Utils; +import com.yahoo.vespa.config.server.Tenants; + +/** + * Handler to create, get and delete a tenant. + * + * @author vegardh + */ +public class TenantHandler extends HttpHandler { + + private static final String TENANT_NAME_REGEXP = "[\\w-]+"; + private final Tenants tenants; + + public TenantHandler(Executor executor, AccessLog accessLog, Tenants tenants) { + super(executor, accessLog); + this.tenants = tenants; + } + + @Override + protected HttpResponse handlePUT(HttpRequest request) { + TenantName tenant = getAndValidateTenantFromRequest(request); + try { + tenants.createTenant(tenant); + } catch (Exception e) { + throw new InternalServerException(Exceptions.toMessageString(e)); + } + return new TenantCreateResponse(tenant); + } + + /** + * Gets the tenant name from the request, throws if it exists already and validates its name + * + * @param request an {@link com.yahoo.container.jdisc.HttpRequest} + * @return tenant name + */ + private TenantName getAndValidateTenantFromRequest(HttpRequest request) { + TenantName tenant = Utils.getTenantFromRequest(request); + Utils.checkThatTenantDoesNotExist(tenants, tenant); + validateTenantName(tenant); + return tenant; + } + + private void validateTenantName(TenantName tenant) { + if (!tenant.value().matches(TENANT_NAME_REGEXP)) { + throw new BadRequestException("Illegal tenant name: " + tenant); + } + } + + @Override + protected HttpResponse handleGET(HttpRequest request) { + TenantName tenant = getExistingTenant(request); + return new TenantGetResponse(tenant); + } + + @Override + protected HttpResponse handleDELETE(HttpRequest request) { + TenantName tenantName = getExistingTenant(request); + Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); + ApplicationRepo applicationRepo = tenant.getApplicationRepo(); + final List<ApplicationId> activeApplications = applicationRepo.listApplications(); + if (activeApplications.isEmpty()) { + try { + tenants.deleteTenant(tenantName); + } catch (Exception e) { + throw new InternalServerException(Exceptions.toMessageString(e)); + } + } else { + throw new BadRequestException("Cannot delete tenant '" + tenantName + "', as it has active applications: " + + activeApplications); + } + return new TenantDeleteResponse(tenantName); + } + + private TenantName getExistingTenant(HttpRequest request) { + TenantName tenant = Utils.getTenantFromRequest(request); + Utils.checkThatTenantExists(tenants, tenant); + return tenant; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java new file mode 100644 index 00000000000..5ba6d480871 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantRequest.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.ApplicationId; + +/** + * Config REST requests that have been bound to an application id + * + * @author vegardh + */ +public interface TenantRequest { + + ApplicationId getApplicationId(); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java new file mode 100644 index 00000000000..318a3f81d52 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/ElkProducer.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; +import com.yahoo.cloud.config.ElkConfig.Builder; + +import com.yahoo.cloud.config.ElkConfig; +import com.yahoo.vespa.defaults.Defaults; + +/** + * Produces the ELK config for the SuperModel + * + * @author vegardh + * @since 5.38 + * + */ +public class ElkProducer implements ElkConfig.Producer { + + private final ElkConfig config; + + public ElkProducer(ElkConfig config) { + this.config = config; + } + + @Override + public void getConfig(Builder builder) { + for (ElkConfig.Elasticsearch es : config.elasticsearch()) { + int port = es.port() != 0 ? es.port() : Defaults.getDefaults().vespaWebServicePort(); + builder.elasticsearch(new ElkConfig.Elasticsearch.Builder().host(es.host()).port(port)); + } + ElkConfig.Logstash.Builder logstashBuilder = new ElkConfig.Logstash.Builder(); + logstashBuilder. + config_file(Defaults.getDefaults().underVespaHome(config.logstash().config_file())). + source_field(config.logstash().source_field()). + spool_size(config.logstash().spool_size()); + ElkConfig.Logstash.Network.Builder networkBuilder = new ElkConfig.Logstash.Network.Builder(). + timeout(config.logstash().network().timeout()); + for (ElkConfig.Logstash.Network.Servers srv : config.logstash().network().servers()) { + networkBuilder. + servers(new ElkConfig.Logstash.Network.Servers.Builder(). + host(srv.host()). + port(srv.port())); + } + logstashBuilder.network(networkBuilder); + for (ElkConfig.Logstash.Files files : config.logstash().files()) { + logstashBuilder.files(new ElkConfig.Logstash.Files.Builder(). + paths(files.paths()). + fields(files.fields())); + } + builder.logstash(logstashBuilder); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java new file mode 100644 index 00000000000..d8f3c0da8d4 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import com.google.common.base.Joiner; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; + +/** + * Produces lb-services cfg + * + * @author vegardh + * @since 5.9 + */ +public class LbServicesProducer implements LbServicesConfig.Producer { + + private final Map<TenantName, Map<ApplicationId, Application>> models; + private final Zone zone; + + public LbServicesProducer(Map<TenantName, Map<ApplicationId, Application>> models, Zone zone) { + this.models = models; + this.zone = zone; + } + + @Override + public void getConfig(LbServicesConfig.Builder builder) { + models.keySet().stream() + .sorted() + .forEach(tenant -> { + builder.tenants(tenant.value(), getTenantConfig(models.get(tenant))); + }); + } + + private LbServicesConfig.Tenants.Builder getTenantConfig(Map<ApplicationId, Application> apps) { + LbServicesConfig.Tenants.Builder tb = new LbServicesConfig.Tenants.Builder(); + apps.keySet().stream() + .sorted() + .forEach(applicationId -> { + tb.applications(createLbAppIdKey(applicationId), getAppConfig(apps.get(applicationId))); + }); + return tb; + } + + private String createLbAppIdKey(ApplicationId applicationId) { + return applicationId.application().value() + ":" + zone.environment().value() + ":" + zone.region().value() + ":" + applicationId.instance().value(); + } + + private LbServicesConfig.Tenants.Applications.Builder getAppConfig(Application app) { + LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); + ab.activeRotation(getActiveRotation(app)); + app.getModel().getHosts().stream() + .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) + .forEach(hostInfo -> { + ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo)); + }); + return ab; + } + + private boolean getActiveRotation(Application app) { + boolean activeRotation = false; + for (HostInfo hostInfo : app.getModel().getHosts()) { + final Optional<ServiceInfo> container = hostInfo.getServices().stream().filter( + serviceInfo -> serviceInfo.getServiceType().equals("container") || + serviceInfo.getServiceType().equals("qrserver")). + findAny(); + if (container.isPresent()) { + activeRotation |= Boolean.valueOf(container.get().getProperty("activeRotation").get()); + } + } + return activeRotation; + } + + private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) { + LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder(); + hb.hostname(hostInfo.getHostname()); + hostInfo.getServices().stream() + .forEach(serviceInfo -> { + hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo)); + }); + return hb; + } + + private LbServicesConfig.Tenants.Applications.Hosts.Services.Builder getServiceConfig(ServiceInfo serviceInfo) { + final List<String> endpointAliases = Stream.of(serviceInfo.getProperty("endpointaliases").orElse("").split(",")). + filter(prop -> !"".equals(prop)).collect(Collectors.toList()); + endpointAliases.addAll(Stream.of(serviceInfo.getProperty("rotations").orElse("").split(",")).filter(prop -> !"".equals(prop)).collect(Collectors.toList())); + Collections.sort(endpointAliases); + + LbServicesConfig.Tenants.Applications.Hosts.Services.Builder sb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Builder() + .type(serviceInfo.getServiceType()) + .clustertype(serviceInfo.getProperty("clustertype").orElse("")) + .clustername(serviceInfo.getProperty("clustername").orElse("")) + .configId(serviceInfo.getConfigId()) + .servicealiases(Stream.of(serviceInfo.getProperty("servicealiases").orElse("").split(",")). + filter(prop -> !"".equals(prop)).sorted((a, b) -> a.compareTo(b)).collect(Collectors.toList())) + .endpointaliases(endpointAliases) + .index(Integer.parseInt(serviceInfo.getProperty("index").orElse("999999"))); + serviceInfo.getPorts().stream() + .forEach(portInfo -> { + LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder() + .number(portInfo.getPort()) + .tags(Joiner.on(" ").join(portInfo.getTags())); + sb.ports(pb); + }); + return sb; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java new file mode 100755 index 00000000000..6a63269ae6e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; + +import com.yahoo.cloud.config.RoutingConfig; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.Application; + +import java.util.Map; + +/** + * Create global config based on info from the zone application + * + * @author can + * @since 5.60 + */ +public class RoutingProducer implements RoutingConfig.Producer { + + private final Map<TenantName, Map<ApplicationId, Application>> models; + + public RoutingProducer(Map<TenantName, Map<ApplicationId, Application>> models) { + this.models = models; + } + + @Override + public void getConfig(RoutingConfig.Builder builder) { + for (Map<ApplicationId, Application> model : models.values()) { + model.values().stream().filter(application -> application.getId().isHostedVespaRoutingApplication()).forEach(application -> { + for (HostInfo host : application.getModel().getHosts()) { + builder.hosts(host.getHostname()); + } + }); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java new file mode 100755 index 00000000000..2be7860b01f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; + +import com.yahoo.cloud.config.ElkConfig; +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.RoutingConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.server.application.Application; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * A config model that spans across all applications of all tenants in the config server. + * + * @author vegardh + * @since 5.9 + * + */ +public class SuperModel implements LbServicesConfig.Producer, ElkConfig.Producer, RoutingConfig.Producer { + + private final Map<TenantName, Map<ApplicationId, Application>> models; + private final LbServicesProducer lbProd; + private final ElkProducer elkProd; + private final RoutingProducer zoneProd; + + public SuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels, ElkConfig elkConfig, Zone zone) { + this.models = newModels; + this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(models), zone); + this.elkProd = new ElkProducer(elkConfig); + this.zoneProd = new RoutingProducer(Collections.unmodifiableMap(models)); + } + + public ConfigPayload getConfig(ConfigKey<?> configKey) throws IOException { + // TODO: Override not applied, but not really necessary here + if (configKey.equals(new ConfigKey<>(LbServicesConfig.class, configKey.getConfigId()))) { + LbServicesConfig.Builder builder = new LbServicesConfig.Builder(); + getConfig(builder); + return ConfigPayload.fromInstance(new LbServicesConfig(builder)); + } else if (configKey.equals(new ConfigKey<>(ElkConfig.class, configKey.getConfigId()))) { + ElkConfig.Builder builder = new ElkConfig.Builder(); + getConfig(builder); + return ConfigPayload.fromInstance(new ElkConfig(builder)); + } else if (configKey.equals(new ConfigKey<>(RoutingConfig.class, configKey.getConfigId()))) { + RoutingConfig.Builder builder = new RoutingConfig.Builder(); + getConfig(builder); + return ConfigPayload.fromInstance(new RoutingConfig(builder)); + } else { + return null; + } + } + + public Map<TenantName, Map<ApplicationId, Application>> getCurrentModels() { + return models; + } + + @Override + public void getConfig(LbServicesConfig.Builder builder) { + lbProd.getConfig(builder); + } + + @Override + public void getConfig(com.yahoo.cloud.config.ElkConfig.Builder builder) { + elkProd.getConfig(builder); + } + + @Override + public void getConfig(RoutingConfig.Builder builder) { + zoneProd.getConfig(builder); + } + + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + TenantName tenant = applicationId.tenant(); + if (!models.containsKey(tenant)) { + throw new IllegalArgumentException("Tenant " + tenant + " not found"); + } + Map<ApplicationId, Application> applications = models.get(tenant); + if (!applications.containsKey(applicationId)) { + throw new IllegalArgumentException("Application id " + applicationId + " not found"); + } + Application application = applications.get(applicationId); + ConfigKey<CONFIGTYPE> key = new ConfigKey<>(configClass, configId); + ConfigPayload payload = application.getModel().getConfig(key, (ConfigDefinition)null, null); + return payload.toInstance(configClass, configId); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java new file mode 100644 index 00000000000..c3046887b0e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.modelfactory; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.RotationsCache; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.deploy.ModelContextImpl; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.StaticProvisioner; +import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; +import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.curator.Curator; + +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Builds activated versions of the right model versions + * + * @author bratseth + */ +public class ActivatedModelsBuilder extends ModelsBuilder<Application> { + + private static final Logger log = Logger.getLogger(ActivatedModelsBuilder.class.getName()); + + private final TenantName tenant; + private final long appGeneration; + private final SessionZooKeeperClient zkClient; + private final Optional<PermanentApplicationPackage> permanentApplicationPackage; + private final Optional<com.yahoo.config.provision.Provisioner> hostProvisioner; + private final ConfigserverConfig configserverConfig; + private final ConfigDefinitionRepo configDefinitionRepo; + private final Metrics metrics; + private final Curator curator; + private final Zone zone; + private final DeployLogger logger; + + public ActivatedModelsBuilder(TenantName tenant, long appGeneration, SessionZooKeeperClient zkClient, GlobalComponentRegistry globalComponentRegistry) { + super(globalComponentRegistry.getModelFactoryRegistry()); + this.tenant = tenant; + this.appGeneration = appGeneration; + this.zkClient = zkClient; + this.permanentApplicationPackage = Optional.of(globalComponentRegistry.getPermanentApplicationPackage()); + this.configserverConfig = globalComponentRegistry.getConfigserverConfig(); + this.configDefinitionRepo = globalComponentRegistry.getConfigDefinitionRepo(); + this.metrics = globalComponentRegistry.getMetrics(); + this.hostProvisioner = globalComponentRegistry.getHostProvisioner(); + this.curator = globalComponentRegistry.getCurator(); + this.zone = globalComponentRegistry.getZone(); + this.logger = new SilentDeployLogger(); + } + + @Override + protected Application buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, + ApplicationId applicationId) { + Version version = modelFactory.getVersion(); + log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", + version, appGeneration, applicationId)); + ModelContext modelContext = new ModelContextImpl( + applicationPackage, + Optional.<Model>empty(), + permanentApplicationPackage.get().applicationPackage(), + logger, + configDefinitionRepo, + getForVersionOrLatest(applicationPackage.getFileRegistryMap(), modelFactory.getVersion()).orElse(new MockFileRegistry()), + createHostProvisioner(getForVersionOrLatest(applicationPackage.getProvisionInfoMap(), modelFactory.getVersion())), + createModelContextProperties(applicationId), + Optional.empty(), + Optional.empty()); + ServerCache cache = zkClient.loadServerCache(); + MetricUpdater applicationMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(applicationId)); + return new Application(modelFactory.createModel(modelContext), cache, appGeneration, version, + applicationMetricUpdater, applicationId); + } + + private Optional<HostProvisioner> createHostProvisioner(Optional<ProvisionInfo> provisionInfo) { + if (hostProvisioner.isPresent() && provisionInfo.isPresent()) { + return Optional.of(createStaticProvisioner(provisionInfo.get())); + } + return Optional.empty(); + } + + private HostProvisioner createStaticProvisioner(ProvisionInfo provisionInfo) { + return new StaticProvisioner(provisionInfo); + } + + private static <T> Optional<T> getForVersionOrLatest(Map<Version, T> map, Version version) { + if (map.isEmpty()) { + return Optional.empty(); + } + T value = map.get(version); + if (value == null) { + value = map.get(map.keySet().stream().max((a, b) -> a.compareTo(b)).get()); + } + return Optional.of(value); + } + + private ModelContext.Properties createModelContextProperties(ApplicationId applicationId) { + return createModelContextProperties( + applicationId, + configserverConfig, + zone, + new RotationsCache(curator, Tenants.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId)); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java new file mode 100644 index 00000000000..a18a200f39e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.modelfactory; + +import com.google.inject.Inject; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; + +import java.util.*; + +/** + * A registry of model factories. Allows querying for a specific version of a {@link ModelFactory} or + * simply returning all of them. Keeps track of the latest {@link com.yahoo.config.provision.Version} supported. + * + * @author lulf + */ +public class ModelFactoryRegistry { + + private final Map<Version, ModelFactory> factories = new HashMap<>(); + + @Inject + public ModelFactoryRegistry(ComponentRegistry<ModelFactory> factories) { + this(factories.allComponents()); + } + + public ModelFactoryRegistry(List<ModelFactory> modelFactories) { + if (modelFactories.isEmpty()) { + throw new IllegalArgumentException("No ModelFactory instances registered, cannot build config models"); + } + for (ModelFactory factory : modelFactories) { + factories.put(factory.getVersion(), factory); + } + } + + public Set<Version> allVersions() { return factories.keySet(); } + + /** + * Returns the factory for the given version + * + * @throws UnknownVespaVersionException if there is no factory for this version + */ + public ModelFactory getFactory(Version version) { + if ( ! factories.containsKey(version)) + throw new UnknownVespaVersionException("Unknown Vespa version '" + version + "', cannot build config model for this version"); + return factories.get(version); + } + + /** + * Return all factories that can build a model. + * + * @return An immutable collection of {@link com.yahoo.config.model.api.ModelFactory} instances. + */ + public Collection<ModelFactory> getFactories() { + return Collections.unmodifiableCollection(factories.values()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelResult.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelResult.java new file mode 100644 index 00000000000..9c9ec160a95 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelResult.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.modelfactory; + +import com.yahoo.config.model.api.Model; + +/** + * @author bratseth + */ +public interface ModelResult { + + Model getModel(); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java new file mode 100644 index 00000000000..99036ee0027 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.modelfactory; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.config.server.deploy.ModelContextImpl; +import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Responsible for building the right versions of application models for a given tenant and application generation. + * Actual model building is implemented by subclasses because it differs in the prepare and activate phases. + * + * @author bratseth + */ +public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { + + private static final Logger log = Logger.getLogger(ModelsBuilder.class.getName()); + + private final ModelFactoryRegistry modelFactoryRegistry; + + protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry) { + this.modelFactoryRegistry = modelFactoryRegistry; + } + + public List<MODELRESULT> buildModels(ApplicationId applicationId, ApplicationPackage applicationPackage) { + Set<Version> versions = modelFactoryRegistry.allVersions(); + + // If the application specifies a major, load models only for that + Optional<Integer> requestedMajorVersion = applicationPackage.getMajorVersion(); + if (requestedMajorVersion.isPresent()) + versions = filterByMajorVersion(requestedMajorVersion.get(), versions); + + // Load models by one major version at the time as new major versions are allowed to be unloadable + // in the case where an existing application is incompatible with a new major version + // (which is possible by the definition of major) + List<Integer> majorVersions = versions.stream() + .map(Version::getMajor) + .distinct() + .sorted(Comparator.reverseOrder()) + .collect(Collectors.toList()); + List<MODELRESULT> allApplicationModels = new ArrayList<>(); + for (int i = 0; i < majorVersions.size(); i++) { + try { + allApplicationModels.addAll(buildModelVersion(filterByMajorVersion(majorVersions.get(i), versions), + applicationId, applicationPackage)); + + // skip old config models after we have found a major version which works + if (allApplicationModels.size() > 0 && allApplicationModels.get(0).getModel().skipOldConfigModels()) + break; + } + catch (RuntimeException e) { // TODO: Make this a specialized exception + boolean isOldestMajor = i == majorVersions.size() - 1; + if (isOldestMajor) { + if (e instanceof NoSuchElementException && "No value present".equals(e.getMessage())) { + e.printStackTrace(); + } + throw new IllegalArgumentException(applicationId + ": Error loading model", e); + } else { + log.log(Level.INFO, applicationId + ": Skipping major version " + majorVersions.get(i), e); + } + } + } + return allApplicationModels; + } + + private List<MODELRESULT> buildModelVersion(Set<Version> versions, ApplicationId applicationId, + ApplicationPackage applicationPackage) { + Version latest = findLatest(versions); + // load latest application version + MODELRESULT latestApplicationVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), applicationPackage, applicationId); + if (latestApplicationVersion.getModel().skipOldConfigModels()) { + return Collections.singletonList(latestApplicationVersion); + } + else { // load old model versions + List<MODELRESULT> allApplicationVersions = new ArrayList<>(); + allApplicationVersions.add(latestApplicationVersion); + for (Version version : versions) { + if (version.equals(latest)) continue; // already loaded + allApplicationVersions.add(buildModelVersion(modelFactoryRegistry.getFactory(version), applicationPackage, applicationId)); + } + return allApplicationVersions; + } + } + + private Set<Version> filterByMajorVersion(int majorVersion, Set<Version> versions) { + Set<Version> filteredVersions = versions.stream().filter(v -> v.getMajor() == majorVersion).collect(Collectors.toSet()); + if (filteredVersions.isEmpty()) + throw new UnknownVespaVersionException("No Vespa versions matching major version " + majorVersion + " are present"); + return filteredVersions; + } + + private Version findLatest(Set<Version> versionSet) { + List<Version> versionList = new ArrayList<>(versionSet); + Collections.sort(versionList); + return versionList.get(versionList.size() - 1); + } + + protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, + ApplicationId applicationId); + + protected ModelContext.Properties createModelContextProperties(ApplicationId applicationId, + ConfigserverConfig configserverConfig, + Zone zone, + Set<Rotation> rotations) { + return new ModelContextImpl.Properties( + applicationId, + configserverConfig.multitenant(), + ConfigServerSpec.fromConfig(configserverConfig), + configserverConfig.hostedVespa(), + zone, + rotations); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java new file mode 100644 index 00000000000..0d6170909a7 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.modelfactory; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.HostValidator; +import com.yahoo.vespa.config.server.RotationsCache; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.deploy.ModelContextImpl; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.provision.ProvisionerAdapter; +import com.yahoo.vespa.config.server.session.FileDistributionFactory; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.session.SessionContext; +import com.yahoo.vespa.curator.Curator; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * @author bratseth + */ +public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.PreparedModelResult> { + + private static final Logger log = Logger.getLogger(PreparedModelsBuilder.class.getName()); + + private final PermanentApplicationPackage permanentApplicationPackage; + private final ConfigserverConfig configserverConfig; + private final ConfigDefinitionRepo configDefinitionRepo; + private final Curator curator; + private final Zone zone; + private final SessionContext context; + private final DeployLogger logger; + private final PrepareParams params; + private final FileDistributionFactory fileDistributionFactory; + private final HostProvisionerProvider hostProvisionerProvider; + private final Optional<ApplicationSet> currentActiveApplicationSet; + private final ApplicationId applicationId; + private final RotationsCache rotationsCache; + private final Set<Rotation> rotations; + private final ModelContext.Properties properties; + + /** Construct from global component registry */ + public PreparedModelsBuilder(GlobalComponentRegistry globalComponentRegistry, + FileDistributionFactory fileDistributionFactory, + HostProvisionerProvider hostProvisionerProvider, + SessionContext context, + DeployLogger logger, + PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, + Path tenantPath) { + super(globalComponentRegistry.getModelFactoryRegistry()); + this.permanentApplicationPackage = globalComponentRegistry.getPermanentApplicationPackage(); + this.configserverConfig = globalComponentRegistry.getConfigserverConfig(); + this.configDefinitionRepo = globalComponentRegistry.getConfigDefinitionRepo(); + this.curator = globalComponentRegistry.getCurator(); + this.zone = globalComponentRegistry.getZone(); + + this.fileDistributionFactory = fileDistributionFactory; + this.hostProvisionerProvider = hostProvisionerProvider; + + this.context = context; + this.logger = logger; + this.params = params; + this.currentActiveApplicationSet = currentActiveApplicationSet; + + this.applicationId = params.getApplicationId(); + this.rotationsCache = new RotationsCache(curator, tenantPath); + this.rotations = getRotations(params.rotations()); + this.properties = createModelContextProperties( + params.getApplicationId(), + configserverConfig, + zone, + rotations); + } + + /** Construct with all dependencies passed separately */ + public PreparedModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, + PermanentApplicationPackage permanentApplicationPackage, + ConfigserverConfig configserverConfig, + ConfigDefinitionRepo configDefinitionRepo, + Curator curator, + Zone zone, + FileDistributionFactory fileDistributionFactory, + HostProvisionerProvider hostProvisionerProvider, + SessionContext context, + DeployLogger logger, + PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, + Path tenantPath) { + super(modelFactoryRegistry); + this.permanentApplicationPackage = permanentApplicationPackage; + this.configserverConfig = configserverConfig; + this.configDefinitionRepo = configDefinitionRepo; + this.curator = curator; + this.zone = zone; + + this.fileDistributionFactory = fileDistributionFactory; + this.hostProvisionerProvider = hostProvisionerProvider; + + this.context = context; + this.logger = logger; + this.params = params; + this.currentActiveApplicationSet = currentActiveApplicationSet; + + this.applicationId = params.getApplicationId(); + this.rotationsCache = new RotationsCache(curator, tenantPath); + this.rotations = getRotations(params.rotations()); + this.properties = new ModelContextImpl.Properties( + params.getApplicationId(), + configserverConfig.multitenant(), + ConfigServerSpec.fromConfig(configserverConfig), + configserverConfig.hostedVespa(), + zone, + rotations); + } + + @Override + protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, + ApplicationId applicationId) { + Version version = modelFactory.getVersion(); + log.log(LogLevel.DEBUG, "Start building model for Vespa version " + version); + FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider( + context.getServerDBSessionDir(), + applicationId); + + Optional<HostProvisioner> hostProvisioner = createHostProvisionerAdapter(properties); + Optional<Model> previousModel = currentActiveApplicationSet + .map(set -> set.getForVersionOrLatest(Optional.of(version)).getModel()); + ModelContext modelContext = new ModelContextImpl( + applicationPackage, + previousModel, + permanentApplicationPackage.applicationPackage(), + logger, + configDefinitionRepo, + fileDistributionProvider.getFileRegistry(), + hostProvisioner, + properties, + getAppDir(applicationPackage), + Optional.of(version)); + + log.log(LogLevel.DEBUG, "Running createAndValidateModel for Vespa version " + version); + ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, params.ignoreValidationErrors()); + validateModelHosts(context.getHostValidator(), applicationId, result.getModel()); + log.log(LogLevel.DEBUG, "Done building model for Vespa version " + version); + return new PreparedModelsBuilder.PreparedModelResult(version, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); + } + + private Optional<File> getAppDir(ApplicationPackage applicationPackage) { + try { + return applicationPackage instanceof FilesApplicationPackage ? + Optional.of(((FilesApplicationPackage) applicationPackage).getAppDir()) : + Optional.empty(); + } catch (IOException e) { + throw new RuntimeException("Could not find app dir", e); + } + } + + private void validateModelHosts(HostValidator<ApplicationId> hostValidator, ApplicationId applicationId, Model model) { + hostValidator.verifyHosts(applicationId, model.getHosts().stream().map(hostInfo -> hostInfo.getHostname()) + .collect(Collectors.toList())); + } + + private Set<Rotation> getRotations(Set<Rotation> rotations) { + if (rotations == null || rotations.isEmpty()) { + rotations = rotationsCache.readRotationsFromZooKeeper(applicationId); + } + return rotations; + } + + private Optional<HostProvisioner> createHostProvisionerAdapter(ModelContext.Properties properties) { + return hostProvisionerProvider.getHostProvisioner().map( + provisioner -> new ProvisionerAdapter(provisioner, properties.applicationId())); + } + + + /** The result of preparing a single model version */ + public static class PreparedModelResult implements ModelResult { + + public final Version version; + public final Model model; + public final FileDistributionProvider fileDistributionProvider; + public final List<ConfigChangeAction> actions; + + public PreparedModelResult(Version version, Model model, + FileDistributionProvider fileDistributionProvider, List<ConfigChangeAction> actions) { + this.version = version; + this.model = model; + this.fileDistributionProvider = fileDistributionProvider; + this.actions = actions; + } + + @Override + public Model getModel() { + return model; + } + + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/MetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/MetricUpdater.java new file mode 100644 index 00000000000..8eae14b9aac --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/MetricUpdater.java @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.monitoring; + +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.RequestHandler; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static com.yahoo.vespa.config.server.monitoring.Metrics.getMetricName; +/** + * @author musum + */ +// TODO javadoc, thread non-safeness maybe +public class MetricUpdater { + private static final String METRIC_UNKNOWN_HOSTS = getMetricName("unknownHostRequests"); + private static final String METRIC_SESSION_CHANGE_ERRORS = getMetricName("sessionChangeErrors"); + private static final String METRIC_NEW_SESSIONS = getMetricName("newSessions"); + private static final String METRIC_PREPARED_SESSIONS = getMetricName("preparedSessions"); + private static final String METRIC_ACTIVATED_SESSIONS = getMetricName("activeSessions"); + private static final String METRIC_DEACTIVATED_SESSIONS = getMetricName("inactiveSessions"); + private static final String METRIC_ADDED_SESSIONS = getMetricName("addedSessions"); + private static final String METRIC_REMOVED_SESSIONS = getMetricName("removedSessions"); + private static final String METRIC_ZK_CONNECTION_LOST = getMetricName("zkConnectionLost"); + private static final String METRIC_ZK_RECONNECTED = getMetricName("zkReconnected"); + private static final String METRIC_ZK_CONNECTED = getMetricName("zkConnected"); + private static final String METRIC_ZK_SUSPENDED = getMetricName("zkSuspended"); + private static final String METRIC_TENANTS = getMetricName("tenants"); + private static final String METRIC_HOSTS = getMetricName("hosts"); + private static final String METRIC_APPLICATIONS = getMetricName("applications"); + private static final String METRIC_CACHE_CONFIG_ELEMENTS = getMetricName("cacheConfigElems"); + private static final String METRIC_CACHE_CONFIG_CHECKSUMS = getMetricName("cacheChecksumElems"); + private static final String METRIC_DELAYED_RESPONSES = getMetricName("delayedResponses"); + private static final String METRIC_RPCSERVER_WORK_QUEUE_SIZE = getMetricName("rpcServerWorkQueueSize"); + + + private final Metrics metrics; + private final Map<String, String> dimensions; + private final Metric.Context metricContext; + private final Map<String, Number> staticMetrics = new ConcurrentHashMap<>(); + + public MetricUpdater(Metrics metrics, Map<String, String> dimensions) { + this.metrics = metrics; + this.dimensions = dimensions; + metricContext = createContext(metrics, dimensions); + } + + public void incrementRequests() { + metrics.incrementRequests(metricContext); + } + + public void incrementFailedRequests() { + metrics.incrementFailedRequests(metricContext); + } + + public void incrementProcTime(long increment) { + metrics.incrementProcTime(increment, metricContext); + } + + /** + * Sets the count for number of config elements in the {@link ServerCache} + * + * @param elems number of elements + */ + public void setCacheConfigElems(long elems) { + staticMetrics.put(METRIC_CACHE_CONFIG_ELEMENTS, elems); + } + + /** + * Sets the count for number of checksum elements in the {@link ServerCache} + * + * @param elems number of elements + */ + public void setCacheChecksumElems(long elems) { + staticMetrics.put(METRIC_CACHE_CONFIG_CHECKSUMS, elems); + } + + /** + * Sets the number of outstanding responses (unchanged config in long poll) + * + * @param elems number of elements + */ + public void setDelayedResponses(long elems) { + staticMetrics.put(METRIC_DELAYED_RESPONSES, elems); + } + + private void setStaticMetric(String name, int size) { + staticMetrics.put(name, size); + } + + /** + * Increment the number of requests where we were unable to map host to a {@link RequestHandler}. + */ + public void incUnknownHostRequests() { + metrics.increment(METRIC_UNKNOWN_HOSTS, metricContext); + } + + private Metric.Context createContext(Metrics metrics, Map<String, String> dimensions) { + if (metrics == null) return null; + + return metrics.getMetric().createContext(dimensions); + } + + public Map<String, Number> getStaticMetrics() { + return staticMetrics; + } + + public Metric.Context getMetricContext() { + return metricContext; + } + + public Map<String, String> getDimensions() { + return dimensions; + } + + /** + * Increment the number of errors from changed sessions. + */ + public void incSessionChangeErrors() { + metrics.increment(METRIC_SESSION_CHANGE_ERRORS, metricContext); + } + + /** + * Set the number of new sessions. + */ + public void setNewSessions(int numNew) { + setStaticMetric(METRIC_NEW_SESSIONS, numNew); + } + + /** + * Set the number of prepared sessions. + */ + public void setPreparedSessions(int numPrepared) { + setStaticMetric(METRIC_PREPARED_SESSIONS, numPrepared); + } + + /** + * Set the number of activated sessions. + */ + public void setActivatedSessions(int numActivated) { + setStaticMetric(METRIC_ACTIVATED_SESSIONS, numActivated); + } + + /** + * Set the number of deactivated sessions. + */ + public void setDeactivatedSessions(int numDeactivated) { + setStaticMetric(METRIC_DEACTIVATED_SESSIONS, numDeactivated); + } + + /** + * Increment the number of removed sessions. + */ + public void incRemovedSessions() { + metrics.increment(METRIC_REMOVED_SESSIONS, metricContext); + } + + /** + * Increment the number of added sessions. + */ + public void incAddedSessions() { + metrics.increment(METRIC_ADDED_SESSIONS, metricContext); + } + + public static MetricUpdater createTestUpdater() { + return new MetricUpdater(Metrics.createTestMetrics(), null); + } + + /** + * Increment the number of ZK connection losses. + */ + public void incZKConnectionLost() { + metrics.increment(METRIC_ZK_CONNECTION_LOST, metricContext); + } + + /** + * Increment the number of ZK connection establishments. + */ + public void incZKConnected() { + metrics.increment(METRIC_ZK_CONNECTED, metricContext); + } + + /** + * Increment the number of ZK connection suspended. + */ + public void incZKSuspended() { + metrics.increment(METRIC_ZK_SUSPENDED, metricContext); + } + + /** + * Increment the number of ZK reconnections. + */ + public void incZKReconnected() { + metrics.increment(METRIC_ZK_RECONNECTED, metricContext); + } + + /** + * Set the number of tenants. + */ + public void setTenants(int numTenants) { + setStaticMetric(METRIC_TENANTS, numTenants); + } + + /** + * Set the number of hosts. + */ + public void setHosts(int numHosts) { + setStaticMetric(METRIC_HOSTS, numHosts); + } + + /** + * Set the number of applications. + */ + public void setApplications(int numApplications) { + setStaticMetric(METRIC_APPLICATIONS, numApplications); + } + + public void setRpcServerQueueSize(int numQueued) { + metrics.set(METRIC_RPCSERVER_WORK_QUEUE_SIZE, numQueued, metricContext); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/MetricUpdaterFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/MetricUpdaterFactory.java new file mode 100644 index 00000000000..7f40414c147 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/MetricUpdaterFactory.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.monitoring; + +import java.util.Map; + +/** + * A factory for creating metric updates with a given context. + * + * @author lulf + * @since 5.15 + */ +public interface MetricUpdaterFactory { + MetricUpdater getOrCreateMetricUpdater(Map<String, String> dimensions); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java new file mode 100644 index 00000000000..d0984baefc2 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.monitoring; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.docproc.jdisc.metric.NullMetric; +import com.yahoo.jdisc.Metric; +import com.yahoo.log.LogLevel; +import com.yahoo.statistics.Statistics; +import com.yahoo.statistics.Counter; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Statistics for server. The statistics framework takes care of logging. + * + * @author Harald Musum + * @since 4.2 + */ +public class Metrics extends TimerTask implements MetricUpdaterFactory { + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Metrics.class.getName()); + private static final String METRIC_REQUESTS = getMetricName("requests"); + private static final String METRIC_FAILED_REQUESTS = getMetricName("failedRequests"); + private static final String METRIC_FREE_MEMORY = getMetricName("freeMemory"); + private static final String METRIC_LATENCY = getMetricName("latency"); + + private final Counter requests; + private final Counter failedRequests; + private final Counter procTimeCounter; + private final Metric metric; + + // TODO The map is the key for now + private final Map<Map<String, String>, MetricUpdater> metricUpdaters = new ConcurrentHashMap<>(); + private final Timer timer = new Timer(); + + @Inject + public Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig) { + this.metric = metric; + requests = createCounter(METRIC_REQUESTS, statistics); + failedRequests = createCounter(METRIC_FAILED_REQUESTS, statistics); + procTimeCounter = createCounter("procTime", statistics); + timer.scheduleAtFixedRate(this, 5000, (long) (healthMonitorConfig.snapshot_interval() * 1000)); + log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); + } + + public static Metrics createTestMetrics() { + NullMetric metric = new NullMetric(); + Statistics.NullImplementation statistics = new Statistics.NullImplementation(); + HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); + builder.snapshot_interval(60.0); + return new Metrics(metric, statistics, new HealthMonitorConfig(builder)); + } + + private Counter createCounter(String name, Statistics statistics) { + return new Counter(name, statistics, false); + } + + + void incrementRequests(Metric.Context metricContext) { + requests.increment(1); + metric.add(METRIC_REQUESTS, 1, metricContext); + } + + void incrementFailedRequests(Metric.Context metricContext) { + failedRequests.increment(1); + metric.add(METRIC_FAILED_REQUESTS, 1, metricContext); + } + + void incrementProcTime(long increment, Metric.Context metricContext) { + procTimeCounter.increment(increment); + metric.set(METRIC_LATENCY, increment, metricContext); + } + + public long getRequests() { + return requests.get(); + } + + public Metric getMetric() { + return metric; + } + + public MetricUpdater removeMetricUpdater(Map<String, String> dimensions) { + return metricUpdaters.remove(dimensions); + } + + public static Map<String, String> createDimensions(ApplicationId applicationId) { + final Map<String, String> properties = new LinkedHashMap<>(); + properties.put("tenantName", applicationId.tenant().value()); + properties.put("applicationName", applicationId.application().value()); + properties.put("applicationInstance", applicationId.instance().value()); + return properties; + } + + public static Map<String, String> createDimensions(TenantName tenant) { + final Map<String, String> properties = new LinkedHashMap<>(); + properties.put("tenantName", tenant.value()); + return properties; + } + + public synchronized MetricUpdater getOrCreateMetricUpdater(Map<String, String> dimensions) { + if (metricUpdaters.containsKey(dimensions)) { + return metricUpdaters.get(dimensions); + } + MetricUpdater metricUpdater = new MetricUpdater(this, dimensions); + metricUpdaters.put(dimensions, metricUpdater); + return metricUpdater; + } + + @Override + public void run() { + for (MetricUpdater metricUpdater : metricUpdaters.values()) { + log.log(LogLevel.DEBUG, "Running metric updater for static values for " + metricUpdater.getDimensions()); + for (Map.Entry<String, Number> fixedMetric : metricUpdater.getStaticMetrics().entrySet()) { + log.log(LogLevel.DEBUG, "Setting " + fixedMetric.getKey()); + metric.set(fixedMetric.getKey(), fixedMetric.getValue(), metricUpdater.getMetricContext()); + } + } + setRegularMetrics(); + timer.purge(); + } + + private void setRegularMetrics() { + metric.set(METRIC_FREE_MEMORY, Runtime.getRuntime().freeMemory(), null); + } + + void increment(String metricName, Metric.Context context) { + metric.add(metricName, 1, context); + } + + void set(String metricName, Number value, Metric.Context context) { + metric.set(metricName, value, context); + } + + static String getMetricName(String name) { + return "configserver." + name; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java new file mode 100644 index 00000000000..91429eaefc0 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.provision; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.log.LogLevel; + +import java.util.Optional; +import java.util.logging.Logger; + +/** + * This class is necessary to support both having and not having a host provisioner. We inject + * a component registry here, which then enables us to check whether or not we have a provisioner available. + * + * @author lulf + * @since 5.15 + */ +public class HostProvisionerProvider { + + private static final Logger log = Logger.getLogger(HostProvisionerProvider.class.getName()); + private final Optional<Provisioner> hostProvisioner; + + public HostProvisionerProvider(ComponentRegistry<Provisioner> hostProvisionerRegistry, ConfigserverConfig configserverConfig) { + if (hostProvisionerRegistry.allComponents().isEmpty() || !configserverConfig.hostedVespa()) { + hostProvisioner = Optional.empty(); + } else { + log.log(LogLevel.DEBUG, "Host provisioner injected. Will be used for all deployments"); + hostProvisioner = Optional.of(hostProvisionerRegistry.allComponents().get(0)); + } + } + + private HostProvisionerProvider(ComponentRegistry<Provisioner> componentRegistry) { + this(componentRegistry, new ConfigserverConfig(new ConfigserverConfig.Builder())); + } + + public Optional<Provisioner> getHostProvisioner() { + return hostProvisioner; + } + + // for testing + public static HostProvisionerProvider empty() { + return new HostProvisionerProvider(new ComponentRegistry<>()); + } + + // for testing + public static HostProvisionerProvider withProvisioner(Provisioner provisioner) { + ComponentRegistry<Provisioner> registry = new ComponentRegistry<>(); + registry.register(ComponentId.createAnonymousComponentId("foobar"), provisioner); + return new HostProvisionerProvider(registry, new ConfigserverConfig(new ConfigserverConfig.Builder().hostedVespa(true))); + } + + /** Creates either an empty provider or a provider having the given provisioner */ + public static HostProvisionerProvider from(Optional<Provisioner> provisioner) { + if (provisioner.isPresent()) + return withProvisioner(provisioner.get()); + else + return empty(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java new file mode 100644 index 00000000000..c1884278cd4 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.provision; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; +import com.yahoo.config.provision.Provisioner; + +import java.util.*; + +/** + * A wrapper for {@link Provisioner} to avoid having to expose multitenant + * behavior to the config model. Adapts interface from a {@link HostProvisioner} to a + * {@link Provisioner}. + * + * @author lulf + * @since 5.11 + */ +public class ProvisionerAdapter implements HostProvisioner { + + private final Provisioner provisioner; + private final ApplicationId applicationId; + + public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId) { + this.provisioner = provisioner; + this.applicationId = applicationId; + } + + @Override + public HostSpec allocateHost(String alias) { + throw new UnsupportedOperationException("Allocating a single host in a hosted environment is not possible"); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return provisioner.prepare(applicationId, cluster, capacity, groups, logger); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java new file mode 100644 index 00000000000..0c9575fd834 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.provision; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Host provisioning from an existing {@link ProvisionInfo} instance. + * + * @author bratseth + */ +public class StaticProvisioner implements HostProvisioner { + + private final ProvisionInfo provisionInfo; + + public StaticProvisioner(ProvisionInfo provisionInfo) { + this.provisionInfo = provisionInfo; + } + + @Override + public HostSpec allocateHost(String alias) { + throw new UnsupportedOperationException(); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + List<HostSpec> l = provisionInfo.getHosts().stream() + .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) + .collect(Collectors.toList()); + return l; + } + + private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) { + if (requestedCluster.group().isPresent()) // we are requesting a specific group + return nodeCluster.equals(requestedCluster); + else // we are requesting nodes of all groups in this cluster + return nodeCluster.equalsIgnoringGroup(requestedCluster); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java new file mode 100644 index 00000000000..1e7114957e9 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/StatusResource.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.restapi.impl; + +import com.google.common.annotations.Beta; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.provision.Version; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.http.v2.HttpGetConfigHandler; +import com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler; +import com.yahoo.vespa.config.server.http.v2.HttpListNamedConfigsHandler; +import com.yahoo.vespa.config.server.http.v2.SessionActiveHandler; +import com.yahoo.vespa.config.server.http.v2.SessionContentHandler; +import com.yahoo.vespa.config.server.http.v2.SessionCreateHandler; +import com.yahoo.vespa.config.server.http.v2.SessionPrepareHandler; +import com.yahoo.vespa.config.server.restapi.resources.StatusInformation; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.List; +import java.util.stream.Collectors; + +/** + * A simple status handler that can provide the status of the config server. + * + * @author lulf + * @since 5.1 + */ +@Beta +@Path("/") +@Produces(MediaType.APPLICATION_JSON) +public class StatusResource { + private final ConfigserverConfig configserverConfig; + private final List<String> modelVersions; + + @SuppressWarnings("UnusedParameters") + public StatusResource(@Component SessionCreateHandler create, + @Component SessionContentHandler content, + @Component SessionPrepareHandler prepare, + @Component SessionActiveHandler active, + @Component HttpGetConfigHandler getHandler, + @Component HttpListConfigsHandler listHandler, + @Component HttpListNamedConfigsHandler listNamedHandler, + @Component GlobalComponentRegistry componentRegistry) { + this.configserverConfig = componentRegistry.getConfigserverConfig(); + this.modelVersions = componentRegistry.getModelFactoryRegistry().getFactories().stream() + .map(ModelFactory::getVersion).map(Version::toString).collect(Collectors.toList()); + } + + @GET + public StatusInformation getStatus() { + return new StatusInformation(configserverConfig, modelVersions); + } +}
\ No newline at end of file diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/package-info.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/package-info.java new file mode 100644 index 00000000000..ade3b6b26bc --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/impl/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.server.restapi.impl; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/StatusInformation.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/StatusInformation.java new file mode 100644 index 00000000000..71b819edc8a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/StatusInformation.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.restapi.resources; + +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Status information of config server. Currently needs to convert generated configserver config to a POJO that can + * be serialized to JSON. + * + * @author lulf + * @since 5.21 + */ +public class StatusInformation { + + public ConfigserverConfig configserverConfig; + public List<String> modelVersions; + + public StatusInformation(com.yahoo.cloud.config.ConfigserverConfig configserverConfig, List<String> modelVersions) { + this.configserverConfig = new ConfigserverConfig(configserverConfig); + this.modelVersions = modelVersions; + } + + public static class ConfigserverConfig { + public final int rpcport; + public final int numthreads; + public final String zookeepercfg; + public final Collection<ZooKeeperServer> zookeeeperserver; + public final long zookeeperBarrierTimeout; + public final Collection<String> configModelPluginDir; + public final String configServerDBDir; + public final int maxgetconfigclients; + public final long sessionLifetime; + public final String applicationDirectory; + public final long masterGeneration; + public final boolean multitenant; + public final int numDelayedResponseThreads; + public final com.yahoo.cloud.config.ConfigserverConfig.PayloadCompressionType.Enum payloadCompressionType; + public final boolean useVespaVersionInRequest; + public final String serverId; + public final String region; + public final String environment; + + + public ConfigserverConfig(com.yahoo.cloud.config.ConfigserverConfig configserverConfig) { + this.rpcport = configserverConfig.rpcport(); + this.numthreads = configserverConfig.numthreads(); + this.zookeepercfg = Defaults.getDefaults().underVespaHome(configserverConfig.zookeepercfg()); + this.zookeeeperserver = configserverConfig.zookeeperserver().stream() + .map(zks -> new ZooKeeperServer(zks.hostname(), zks.port())) + .collect(Collectors.toList()); + this.zookeeperBarrierTimeout = configserverConfig.zookeeper().barrierTimeout(); + this.configModelPluginDir = configserverConfig.configModelPluginDir(); + this.configServerDBDir = Defaults.getDefaults().underVespaHome(configserverConfig.configServerDBDir()); + this.maxgetconfigclients = configserverConfig.maxgetconfigclients(); + this.sessionLifetime = configserverConfig.sessionLifetime(); + this.applicationDirectory = Defaults.getDefaults().underVespaHome(configserverConfig.applicationDirectory()); + this.masterGeneration = configserverConfig.masterGeneration(); + this.multitenant = configserverConfig.multitenant(); + this.numDelayedResponseThreads = configserverConfig.numDelayedResponseThreads(); + this.payloadCompressionType = configserverConfig.payloadCompressionType(); + this.useVespaVersionInRequest = configserverConfig.useVespaVersionInRequest(); + this.serverId = configserverConfig.serverId(); + this.region = configserverConfig.region(); + this.environment = configserverConfig.environment(); + } + } + + public static class ZooKeeperServer { + public final String hostname; + public final int port; + + public ZooKeeperServer(String hostname, int port) { + this.hostname = hostname; + this.port = port; + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/package-info.java b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/package-info.java new file mode 100644 index 00000000000..1794c47a6c6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/restapi/resources/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.config.server.restapi.resources; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java new file mode 100644 index 00000000000..9575907ab67 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionLock; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; +import com.yahoo.vespa.curator.Curator; + +import java.io.File; +import java.util.concurrent.locks.Lock; + +/** + * Factory for creating providers that are used to interact with file distribution. + * + * @author lulf + * @since 5.1 + */ +@SuppressWarnings("WeakerAccess") +public class FileDistributionFactory { + + private static final String lockPath = "/vespa/filedistribution/lock"; + private final String zkSpec; + private final Lock lock; + + @Inject + public FileDistributionFactory(Curator curator) { + this(curator, curator.connectionSpec()); + } + + public FileDistributionFactory(Curator curator, String zkSpec) { + this.lock = new FileDistributionLock(curator, lockPath); + this.zkSpec = zkSpec; + } + + public FileDistributionProvider createProvider(File applicationPackage, ApplicationId applicationId) { + return new FileDistributionProvider(applicationPackage, zkSpec, applicationId.serializedForm(), lock); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java new file mode 100644 index 00000000000..d2857068885 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -0,0 +1,170 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.transaction.Transaction; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.curator.Curator; + +import java.io.File; +import java.util.Optional; + +/** + * A LocalSession is a session that has been created locally on this configserver. A local session can be edited and + * prepared. Deleting a local session will ensure that the local filesystem state and global zookeeper state is + * cleaned for this session. + * + * @author lulf + * @since 5.1 + */ +public class LocalSession extends Session implements Comparable<LocalSession> { + + private final ApplicationPackage applicationPackage; + private final ApplicationRepo applicationRepo; + private final SessionZooKeeperClient zooKeeperClient; + private final SessionPreparer sessionPreparer; + private final SessionContext sessionContext; + private final File serverDB; + private final SuperModelGenerationCounter superModelGenerationCounter; + + /** + * Create a session. This involves loading the application, validating it and distributing it. + * + * @param sessionId The session id for this session. + */ + // TODO tenant in SessionContext? + public LocalSession(TenantName tenant, long sessionId, SessionPreparer sessionPreparer, SessionContext sessionContext) { + super(tenant, sessionId); + this.serverDB = sessionContext.getServerDBSessionDir(); + this.applicationPackage = sessionContext.getApplicationPackage(); + this.zooKeeperClient = sessionContext.getSessionZooKeeperClient(); + this.applicationRepo = sessionContext.getApplicationRepo(); + this.sessionPreparer = sessionPreparer; + this.sessionContext = sessionContext; + this.superModelGenerationCounter = sessionContext.getSuperModelGenerationCounter(); + } + + public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath) { + Curator.CompletionWaiter waiter = zooKeeperClient.createPrepareWaiter(); + ConfigChangeActions actions = sessionPreparer.prepare(sessionContext, logger, params, currentActiveApplicationSet, tenantPath); + setPrepared(); + waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); + return actions; + } + + public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { + if (mode.equals(Mode.WRITE)) { + markSessionEdited(); + } + return applicationPackage.getFile(relativePath); + } + + private void setPrepared() { + setStatus(Session.Status.PREPARE); + } + + private Transaction setActive() { + Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); + transaction.add(applicationRepo.createPutApplicationTransaction(zooKeeperClient.readApplicationId(getTenant()), getSessionId()).operations()); + return transaction; + } + + private Transaction createSetStatusTransaction(Status status) { + return zooKeeperClient.createWriteStatusTransaction(status); + } + + public Session.Status getStatus() { + return zooKeeperClient.readStatus(); + } + + private void setStatus(Session.Status newStatus) { + zooKeeperClient.writeStatus(newStatus); + } + + public Transaction createActivateTransaction() { + zooKeeperClient.createActiveWaiter(); + superModelGenerationCounter.increment(); + return setActive(); + } + + public Transaction createDeactivateTransaction() { + return createSetStatusTransaction(Status.DEACTIVATE); + } + + private void markSessionEdited() { + setStatus(Session.Status.NEW); + } + + public long getActiveSessionAtCreate() { + return applicationPackage.getMetaData().getPreviousActiveGeneration(); + } + + // Note: Assumes monotonically increasing session ids + public boolean isNewerThan(long sessionId) { + return getSessionId() > sessionId; + } + + /** + * Deletes this session from ZooKeeper and filesystem, as well as making sure the supermodel generation counter is incremented. + */ + public void delete() { + superModelGenerationCounter.increment(); + IOUtils.recursiveDeleteDir(serverDB); + zooKeeperClient.delete(); + } + + @Override + public int compareTo(LocalSession rhs) { + Long lhsId = getSessionId(); + Long rhsId = rhs.getSessionId(); + return lhsId.compareTo(rhsId); + } + + // in seconds + public long getCreateTime() { + return zooKeeperClient.readCreateTime(); + } + + public void waitUntilActivated(TimeoutBudget timeoutBudget) { + zooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); + } + + public void setApplicationId(ApplicationId applicationId) { + zooKeeperClient.writeApplicationId(applicationId); + } + + public enum Mode { + READ, WRITE + } + + public ApplicationMetaData getMetaData() { + return applicationPackage.getMetaData(); + } + + public ApplicationId getApplicationId() { + return zooKeeperClient.readApplicationId(getTenant()); + } + + public ProvisionInfo getProvisionInfo() { + return zooKeeperClient.getProvisionInfo(); + } + + @Override + public String logPre() { + if (getApplicationId().equals(ApplicationId.defaultId())) { + return Tenants.logPre(getTenant()); + } else { + return Tenants.logPre(getApplicationId()); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionLoader.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionLoader.java new file mode 100644 index 00000000000..3aa8155a7c6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionLoader.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +/** + * Interface of a component that is able to load a session given a session id. + * + * @author lulf + * @since 5.1 + */ +public interface LocalSessionLoader { + + LocalSession loadSession(long sessionId); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java new file mode 100644 index 00000000000..8f39d0b96d1 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.log.LogLevel; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; + +import java.io.File; +import java.io.FilenameFilter; +import java.time.Clock; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * File-based session repository for LocalSessions. Contains state for the local instance of the configserver. + * + * @author lulf + * @since 5.1 + */ +public class LocalSessionRepo extends SessionRepo<LocalSession> { + + private static final Logger log = Logger.getLogger(LocalSessionRepo.class.getName()); + + private final static FilenameFilter sessionApplicationsFilter = new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.matches("\\d+"); + } + }; + + private final long sessionLifetime; // in seconds + private final ApplicationRepo applicationRepo; + private final Clock clock; + + public LocalSessionRepo(TenantFileSystemDirs tenantFileSystemDirs, LocalSessionLoader loader, ApplicationRepo applicationRepo, Clock clock, long sessionLifeTime) { + this(applicationRepo, clock, sessionLifeTime); + loadSessions(tenantFileSystemDirs.path(), loader); + } + + private void loadSessions(File applicationsDir, LocalSessionLoader loader) { + File[] applications = applicationsDir.listFiles(sessionApplicationsFilter); + if (applications == null) { + return; + } + for (File application : applications) { + try { + addSession(loader.loadSession(Long.parseLong(application.getName()))); + } catch (IllegalArgumentException e) { + log.log(LogLevel.WARNING, "Could not load application '" + application.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); + } + } + } + + /** + * Gets the active Session for the given application id. + * + * @return the active session, or null if there is no active session for the given application id. + */ + public LocalSession getActiveSession(ApplicationId applicationId) { + List<ApplicationId> applicationIds = applicationRepo.listApplications(); + if (applicationIds.contains(applicationId)) { + return getSession(applicationRepo.getSessionIdForApplication(applicationId)); + } + return null; + } + + // Constructor only for testing + public LocalSessionRepo(ApplicationRepo applicationRepo, Clock clock, long sessionLifetime) { + this.applicationRepo = applicationRepo; + this.sessionLifetime = sessionLifetime; + this.clock = clock; + } + + public LocalSessionRepo(ApplicationRepo applicationRepo) { + this(applicationRepo, Clock.systemUTC(), TimeUnit.DAYS.toMillis(1)); + } + + @Override + public synchronized void addSession(LocalSession session) { + purgeOldSessions(); + super.addSession(session); + } + + private void purgeOldSessions() { + final List<ApplicationId> applicationIds = applicationRepo.listApplications(); + List<LocalSession> sessions = new ArrayList<>(listSessions()); + for (LocalSession candidate : sessions) { + if (hasExpired(candidate) && !isActiveSession(candidate, applicationIds)) { + deleteSession(candidate); + } + } + } + + private boolean hasExpired(LocalSession candidate) { + return (candidate.getCreateTime() + sessionLifetime) <= TimeUnit.MILLISECONDS.toSeconds(clock.millis()); + } + + private boolean isActiveSession(LocalSession candidate, List<ApplicationId> activeIds) { + if (candidate.getStatus() == Session.Status.ACTIVATE && activeIds.contains(candidate.getApplicationId())) { + long sessionId = applicationRepo.getSessionIdForApplication(candidate.getApplicationId()); + return (candidate.getSessionId() == sessionId); + } else { + return false; + } + } + + private void deleteSession(LocalSession candidate) { + removeSession(candidate.getSessionId()); + candidate.delete(); + } + + public void deleteAllSessions() { + List<LocalSession> sessions = new ArrayList<>(listSessions()); + for (LocalSession session : sessions) { + deleteSession(session); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java new file mode 100644 index 00000000000..6c9224db0fe --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.http.SessionHandler; + +import java.time.Clock; +import java.time.Duration; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +/** + * Parameters for prepare. + * + * @author lulf + * @since 5.1.24 + */ +public final class PrepareParams { + + static final String APPLICATION_NAME_PARAM_NAME = "applicationName"; + static final String INSTANCE_PARAM_NAME = "instance"; + static final String IGNORE_VALIDATION_PARAM_NAME = "ignoreValidationErrors"; + static final String DRY_RUN_PARAM_NAME = "dryRun"; + static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; + static final String ROTATIONS_PARAM_NAME = "rotations"; + static final String DOCKER_VESPA_IMAGE_VERSION_PARAM_NAME = "dockerVespaImageVersion"; + + private boolean ignoreValidationErrors = false; + private boolean dryRun = false; + private ApplicationId applicationId = ApplicationId.defaultId(); + private TimeoutBudget timeoutBudget; + private Optional<Version> vespaVersion = Optional.empty(); + private Set<Rotation> rotations; + private Optional<Version> dockerVespaImageVersion = Optional.empty(); + + PrepareParams() { + this(new ConfigserverConfig(new ConfigserverConfig.Builder())); + } + + public PrepareParams(ConfigserverConfig configserverConfig) { + timeoutBudget = new TimeoutBudget(Clock.systemUTC(), getBarrierTimeout(configserverConfig)); + } + + public PrepareParams applicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + return this; + } + + public PrepareParams ignoreValidationErrors(boolean ignoreValidationErrors) { + this.ignoreValidationErrors = ignoreValidationErrors; + return this; + } + + public PrepareParams dryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + + public PrepareParams timeoutBudget(TimeoutBudget timeoutBudget) { + this.timeoutBudget = timeoutBudget; + return this; + } + + public PrepareParams vespaVersion(String vespaVersion) { + Optional<Version> version = Optional.empty(); + if (vespaVersion != null && !vespaVersion.isEmpty()) { + version = Optional.of(Version.fromString(vespaVersion)); + } + this.vespaVersion = version; + return this; + } + + public PrepareParams rotations(String rotationsString) { + this.rotations = new LinkedHashSet<>(); + if (rotationsString != null && !rotationsString.isEmpty()) { + String[] rotations = rotationsString.split(","); + for (String s : rotations) { + this.rotations.add(new Rotation(s)); + } + } + return this; + } + + public PrepareParams dockerVespaImageVersion(String dockerVespaImageVersion) { + Optional<Version> version = Optional.empty(); + if (dockerVespaImageVersion != null && !dockerVespaImageVersion.isEmpty()) { + version = Optional.of(Version.fromString(dockerVespaImageVersion)); + } + this.dockerVespaImageVersion = version; + return this; + } + + public static PrepareParams fromHttpRequest(HttpRequest request, TenantName tenant, ConfigserverConfig configserverConfig) { + return new PrepareParams(configserverConfig).ignoreValidationErrors(request.getBooleanProperty(IGNORE_VALIDATION_PARAM_NAME)) + .dryRun(request.getBooleanProperty(DRY_RUN_PARAM_NAME)) + .timeoutBudget(SessionHandler.getTimeoutBudget(request, getBarrierTimeout(configserverConfig))) + .applicationId(createApplicationId(request, tenant)) + .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) + .rotations(request.getProperty(ROTATIONS_PARAM_NAME)) + .dockerVespaImageVersion(request.getProperty(DOCKER_VESPA_IMAGE_VERSION_PARAM_NAME)); + } + + private static Duration getBarrierTimeout(ConfigserverConfig configserverConfig) { + return Duration.ofSeconds(configserverConfig.zookeeper().barrierTimeout()); + } + + private static ApplicationId createApplicationId(HttpRequest request, TenantName tenant) { + return new ApplicationId.Builder() + .tenant(tenant) + .applicationName(getPropertyWithDefault(request, APPLICATION_NAME_PARAM_NAME, "default")) + .instanceName(getPropertyWithDefault(request, INSTANCE_PARAM_NAME, "default")) + .build(); + } + + private static String getPropertyWithDefault(HttpRequest request, String propertyName, String defaultProperty) { + return getProperty(request, propertyName).orElse(defaultProperty); + } + + private static Optional<String> getProperty(HttpRequest request, String propertyName) { + return Optional.ofNullable(request.getProperty(propertyName)); + } + + public String getApplicationName() { + return applicationId.application().value(); + } + + public ApplicationId getApplicationId() { + return applicationId; + } + + public Optional<Version> vespaVersion() { return vespaVersion; } + + public Set<Rotation> rotations() { return rotations; } + + public boolean ignoreValidationErrors() { + return ignoreValidationErrors; + } + + public boolean isDryRun() { + return dryRun; + } + + public TimeoutBudget getTimeoutBudget() { + return timeoutBudget; + } + + public Optional<Version> getVespaVersion() { + return vespaVersion; + } + + public Set<Rotation> getRotations() { + return rotations; + } + + public Optional<Version> getDockerVespaImageVersion() { + return dockerVespaImageVersion; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java new file mode 100644 index 00000000000..2ce378d0464 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.provision.*; +import com.yahoo.vespa.config.server.*; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.modelfactory.ActivatedModelsBuilder; +import com.yahoo.vespa.curator.Curator; + +import java.util.*; +import java.util.logging.Logger; + +/** + * A RemoteSession represents a session created on another config server. This session can + * be regarded as read only, and this interface only allows reading information about a session. + * + * @author lulf + * @since 5.1 + */ +public class RemoteSession extends Session { + + private static final Logger log = Logger.getLogger(RemoteSession.class.getName()); + private volatile ApplicationSet applicationSet = null; + private final SessionZooKeeperClient zooKeeperClient; + private final ActivatedModelsBuilder applicationLoader; + + /** + * Creates a session. This involves loading the application, validating it and distributing it. + * + * @param tenant The name of the tenant creating session + * @param sessionId The session id for this session. + * @param globalComponentRegistry a registry of global components + * @param zooKeeperClient a SessionZooKeeperClient instance + */ + public RemoteSession(TenantName tenant, + long sessionId, + GlobalComponentRegistry globalComponentRegistry, + SessionZooKeeperClient zooKeeperClient) { + super(tenant, sessionId); + this.zooKeeperClient = zooKeeperClient; + this.applicationLoader = new ActivatedModelsBuilder(tenant, sessionId, zooKeeperClient, globalComponentRegistry); + } + + public void loadPrepared() { + Curator.CompletionWaiter waiter = zooKeeperClient.getPrepareWaiter(); + ensureApplicationLoaded(); + waiter.notifyCompletion(); + } + + private ApplicationSet loadApplication() { + return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(getTenant()), + zooKeeperClient.loadApplicationPackage())); + } + + public ApplicationSet ensureApplicationLoaded() { + if (applicationSet == null) { + applicationSet = loadApplication(); + } + return applicationSet; + } + + public Session.Status getStatus() { + return zooKeeperClient.readStatus(); + } + + public void deactivate() { + applicationSet = null; + } + + public void makeActive(ReloadHandler reloadHandler) { + Curator.CompletionWaiter waiter = zooKeeperClient.getActiveWaiter(); + log.log(LogLevel.DEBUG, logPre()+"Getting session from repo: " + getSessionId()); + ApplicationSet app = ensureApplicationLoaded(); + log.log(LogLevel.DEBUG, logPre() + "Reloading config for " + app); + reloadHandler.reloadConfig(app); + log.log(LogLevel.DEBUG, logPre() + "Notifying " + waiter); + waiter.notifyCompletion(); + log.log(LogLevel.DEBUG, logPre() + "Session activated: " + app); + } + + @Override + public String logPre() { + if (applicationSet != null) { + return Tenants.logPre(applicationSet.getForVersionOrLatest(Optional.empty()).getId()); + } + + return Tenants.logPre(getTenant()); + } + + public void confirmUpload() { + Curator.CompletionWaiter waiter = zooKeeperClient.getUploadWaiter(); + log.log(LogLevel.DEBUG, "Notifying upload waiter for session " + getSessionId()); + waiter.notifyCompletion(); + log.log(LogLevel.DEBUG, "Done notifying for session " + getSessionId()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java new file mode 100644 index 00000000000..c44436740be --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; + +/** + * @author lulf + * @since 5.1.24 + */ +public class RemoteSessionFactory { + + private final GlobalComponentRegistry componentRegistry; + private final Curator curator; + private final ConfigCurator configCurator; + private final Path sessionDirPath; + private final ConfigDefinitionRepo defRepo; + private final TenantName tenant; + private final ConfigserverConfig configserverConfig; + + public RemoteSessionFactory(GlobalComponentRegistry componentRegistry, + Path sessionsPath, + TenantName tenant) { + this.componentRegistry = componentRegistry; + this.curator = componentRegistry.getCurator(); + this.configCurator = componentRegistry.getConfigCurator(); + this.sessionDirPath = sessionsPath; + this.tenant = tenant; + this.defRepo = componentRegistry.getConfigDefinitionRepo(); + this.configserverConfig = componentRegistry.getConfigserverConfig(); + } + + public RemoteSession createSession(long sessionId) { + Path sessionPath = sessionDirPath.append(String.valueOf(sessionId)); + SessionZooKeeperClient sessionZKClient = new SessionZooKeeperClient(curator, configCurator, sessionPath, defRepo, configserverConfig.serverId()); + return new RemoteSession(tenant, sessionId, componentRegistry, sessionZKClient); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java new file mode 100644 index 00000000000..78d7704506f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.*; + +/** + * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper, set for example + * by the prepare HTTP handler on another configserver. The zookeeper state watched in this class is shared + * between all configservers, so it should not modify any global state, because the operation will be performed + * on all servers. The repo can be regarded as read only from the POV of the configserver. + * + * @author vegardh + * @author lulf + * @since 5.1 + */ +public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements NodeCacheListener, PathChildrenCacheListener { + + private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); + private final Curator curator; + private final Path sessionsPath; + private final RemoteSessionFactory remoteSessionFactory; + private final Map<Long, SessionStateWatcher> sessionStateWatchers = new HashMap<>(); + private final ReloadHandler reloadHandler; + private final MetricUpdater metrics; + private final Curator.DirectoryCache directoryCache; + private final ApplicationRepo applicationRepo; + + public static RemoteSessionRepo create(Curator curator, + RemoteSessionFactory remoteSessionFactory, + ReloadHandler reloadHandler, + Path sessionsPath, + ApplicationRepo applicationRepo, + MetricUpdater metrics, + ExecutorService executorService) throws Exception { + return new RemoteSessionRepo(curator, remoteSessionFactory, reloadHandler, sessionsPath, applicationRepo, metrics, executorService); + } + + /** + * Used when the RemoteSessionRepo is set up programmatically from a Tenant, i.e. config v2 + * @param curator a {@link Curator} instance. + * @param remoteSessionFactory a {@link com.yahoo.vespa.config.server.session.RemoteSessionFactory} + * @param reloadHandler a {@link com.yahoo.vespa.config.server.ReloadHandler} + * @param sessionsPath a {@link com.yahoo.path.Path} to the sessions dir. + * @param applicationRepo an {@link com.yahoo.vespa.config.server.application.ApplicationRepo} object. + * @param executorService an {@link ExecutorService} to run callbacks from ZooKeeper. + * @throws java.lang.Exception if creating the repo fails + */ + private RemoteSessionRepo(Curator curator, + RemoteSessionFactory remoteSessionFactory, + ReloadHandler reloadHandler, + Path sessionsPath, + ApplicationRepo applicationRepo, + MetricUpdater metricUpdater, + ExecutorService executorService) throws Exception { + this.curator = curator; + this.sessionsPath = sessionsPath; + this.applicationRepo = applicationRepo; + this.remoteSessionFactory = remoteSessionFactory; + this.reloadHandler = reloadHandler; + this.metrics = metricUpdater; + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, executorService); + this.directoryCache.start(); + this.directoryCache.addListener(this); + sessionsChanged(getSessionList(directoryCache.getCurrentData())); + } + + private void loadActiveSession(RemoteSession session) { + tryReload(session.ensureApplicationLoaded(), session.logPre()); + } + + private void tryReload(ApplicationSet applicationSet, String logPre) { + try { + reloadHandler.reloadConfig(applicationSet); + log.log(LogLevel.INFO, logPre+"Application activated successfully: " + applicationSet.getId()); + } catch (Exception e) { + log.log(LogLevel.WARNING, logPre+"Skipping loading of application '" + applicationSet.getId() + "': " + Exceptions.toMessageString(e)); + } + } + + // For testing only + public RemoteSessionRepo() { + this.curator = null; + this.remoteSessionFactory = null; + this.reloadHandler = null; + this.sessionsPath = Path.createRoot(); + this.metrics = null; + this.directoryCache = null; + this.applicationRepo = null; + } + + private List<Long> getSessionList(List<ChildData> children) { + List<Long> sessions = new ArrayList<>(); + for (ChildData data : children) { + sessions.add(Long.parseLong(Path.fromString(data.getPath()).getName())); + } + return sessions; + } + + synchronized void sessionsChanged(List<Long> sessions) throws NumberFormatException { + checkForRemovedSessions(sessions); + checkForAddedSessions(sessions); + } + + private void checkForRemovedSessions(List<Long> sessions) { + for (RemoteSession session : listSessions()) { + if (!sessions.contains(session.getSessionId())) { + SessionStateWatcher watcher = sessionStateWatchers.remove(session.getSessionId()); + watcher.close(); + removeSession(session.getSessionId()); + metrics.incRemovedSessions(); + } + } + } + + private void checkForAddedSessions(List<Long> sessions) { + for (Long sessionId : sessions) { + if (getSession(sessionId) == null) { + log.log(LogLevel.DEBUG, "Loading session id " + sessionId); + newSession(sessionId); + metrics.incAddedSessions(); + } + } + } + + /** + * A session for which we don't have a watcher, i.e. hitherto unknown to us. + * + * @param sessionId session id for the new session + */ + private void newSession(long sessionId) { + try { + log.log(LogLevel.DEBUG, "Adding session to RemoteSessionRepo: " + sessionId); + RemoteSession session = remoteSessionFactory.createSession(sessionId); + Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); + Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + fileCache.addListener(this); + loadSessionIfActive(session); + sessionStateWatchers.put(sessionId, new SessionStateWatcher(fileCache, reloadHandler, session, metrics)); + addSession(session); + } catch (Exception e) { + log.log(Level.WARNING, "Failed loading session " + sessionId + " (no config for this session can be served) : " + Exceptions.toMessageString(e)); + } + } + + private void loadSessionIfActive(RemoteSession session) { + for (ApplicationId applicationId : applicationRepo.listApplications()) { + try { + if (applicationRepo.getSessionIdForApplication(applicationId) == session.getSessionId()) { + log.log(LogLevel.DEBUG, "Found active application for session " + session.getSessionId() + " , loading it"); + loadActiveSession(session); + break; + } + } catch (Exception e) { + log.log(LogLevel.WARNING, session.logPre() + " error reading session id for " + applicationId); + } + } + } + + public synchronized void close() { + try { + if (directoryCache != null) { + directoryCache.close(); + } + } catch (Exception e) { + log.log(LogLevel.WARNING, "Exception when closing path cache", e); + } finally { + checkForRemovedSessions(new ArrayList<>()); + } + } + + @Override + public void nodeChanged() throws Exception { + Multiset<Session.Status> sessionMetrics = HashMultiset.create(); + for (RemoteSession session : listSessions()) { + sessionMetrics.add(session.getStatus()); + } + metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); + metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); + metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); + metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); + } + + @Override + public void childEvent(CuratorFramework framework, PathChildrenCacheEvent event) throws Exception { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Got child event: " + event); + } + switch (event.getType()) { + case CHILD_ADDED: + sessionsChanged(getSessionList(directoryCache.getCurrentData())); + synchronizeOnNew(getSessionList(Collections.singletonList(event.getData()))); + break; + case CHILD_REMOVED: + sessionsChanged(getSessionList(directoryCache.getCurrentData())); + break; + } + } + + private void synchronizeOnNew(List<Long> sessionList) { + for (long sessionId : sessionList) { + RemoteSession session = getSession(sessionId); + log.log(LogLevel.DEBUG, session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/ServerCacheLoader.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/ServerCacheLoader.java new file mode 100644 index 00000000000..329eab89458 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/ServerCacheLoader.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.base.Splitter; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.subscription.CfgConfigPayloadBuilder; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +import java.util.Arrays; +import java.util.Map; + +/** + * This class is tasked with reading config definitions and legacy configs/ from zookeeper, and create + * a {@link ServerCache} instance containing these in memory. + * + * @author lulf + * @since 5.1 + */ +public class ServerCacheLoader { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ServerCacheLoader.class.getName()); + private final ConfigDefinitionRepo repo; + private final ConfigCurator configCurator; + private final Path path; + public ServerCacheLoader(ConfigCurator configCurator, Path rootPath, ConfigDefinitionRepo repo) { + this.configCurator = configCurator; + this.path = rootPath; + this.repo = repo; + } + + public ServerCache loadCache() { + return loadConfigDefinitions(); + } + + /** + * Reads config definitions from zookeeper, parses them and puts both ConfigDefinition instances + * and payload (raw config definition) into cache. + * + * @return the populated cache. + */ + public ServerCache loadConfigDefinitions() { + ServerCache cache = new ServerCache(); + try { + log.log(LogLevel.DEBUG, "Getting config definitions"); + loadGlobalConfigDefinitions(cache); + loadConfigDefinitionsFromPath(cache, path.append(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute()); + log.log(LogLevel.DEBUG, "Done getting config definitions"); + } catch (Exception e) { + throw new IllegalStateException("Could not load config definitions for " + path, e); + } + return cache; + } + + private void loadGlobalConfigDefinitions(ServerCache cache) { + for (Map.Entry<ConfigDefinitionKey, ConfigDefinition> entry : repo.getConfigDefinitions().entrySet()) { + cache.addDef(entry.getKey(), entry.getValue()); + } + } + + /** + * Loads config definitions from a specified path into server cache and returns it. + * + * @param appPath the path to load config definitions from + */ + private void loadConfigDefinitionsFromPath(ServerCache cache, String appPath) throws InterruptedException { + if ( ! configCurator.exists(appPath)) return; + for (String nodeName : configCurator.getChildren(appPath)) { + String payload = configCurator.getData(appPath, nodeName); + ConfigDefinitionKey dKey = ConfigUtils.createConfigDefinitionKeyFromZKString(nodeName); + cache.addDef(dKey, new ConfigDefinition(dKey.getName(), Splitter.on("\n").splitToList(payload).toArray(new String[0]))); + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java new file mode 100644 index 00000000000..961f9d10a60 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.Tenants; + +/** + * A session represents an instance of an application that can be edited, prepared and activated. This + * class represents the common stuff between sessions working on the local file + * system ({@link LocalSession}s) and sessions working on zookeeper {@link RemoteSession}s. + * + * @author lulf + * @since 5.1 + */ +public abstract class Session { + + private final long sessionId; + protected final TenantName tenant; + + protected Session(TenantName tenant, long sessionId) { + this.tenant = tenant; + this.sessionId = sessionId; + } + /** + * Retrieve the session id for this session. + * @return the session id. + */ + public final long getSessionId() { + return sessionId; + } + + @Override + public String toString() { + return "Session,id=" + sessionId; + } + + /** + * Represents the status of this session. + */ + public enum Status { + NEW, PREPARE, ACTIVATE, DEACTIVATE, NONE; + + public static Status parse(String data) { + for (Status status : Status.values()) { + if (status.name().equals(data)) { + return status; + } + } + return Status.NEW; + } + } + + public TenantName getTenant() { + return tenant; + } + + /** + * Helper to provide a log message preamble for code dealing with sessions + * @return log preamble + */ + public String logPre() { + return Tenants.logPre(getTenant()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java new file mode 100644 index 00000000000..dd908eaa559 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.config.server.HostValidator; +import com.yahoo.vespa.config.server.SuperModelGenerationCounter; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.ApplicationRepo; + +import java.io.File; + +/** + * The dependencies needed for a local session to be edited and prepared. + * + * @author lulf + * @since 5.1 + */ +public class SessionContext { + + private final ApplicationPackage applicationPackage; + private final SessionZooKeeperClient sessionZooKeeperClient; + private final File serverDBSessionDir; + private final ApplicationRepo applicationRepo; + private final HostValidator<ApplicationId> hostRegistry; + private final SuperModelGenerationCounter superModelGenerationCounter; + + public SessionContext(ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient, + File serverDBSessionDir, ApplicationRepo applicationRepo, + HostValidator<ApplicationId> hostRegistry, SuperModelGenerationCounter superModelGenerationCounter) { + this.applicationPackage = applicationPackage; + this.sessionZooKeeperClient = sessionZooKeeperClient; + this.serverDBSessionDir = serverDBSessionDir; + this.applicationRepo = applicationRepo; + this.hostRegistry = hostRegistry; + this.superModelGenerationCounter = superModelGenerationCounter; + } + + public ApplicationPackage getApplicationPackage() { + return applicationPackage; + } + + public SessionZooKeeperClient getSessionZooKeeperClient() { + return sessionZooKeeperClient; + } + + public File getServerDBSessionDir() { + return serverDBSessionDir; + } + + public ApplicationRepo getApplicationRepo() { + return applicationRepo; + } + + public HostValidator<ApplicationId> getHostValidator() { return hostRegistry; } + + public SuperModelGenerationCounter getSuperModelGenerationCounter() { + return superModelGenerationCounter; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java new file mode 100644 index 00000000000..87af5351186 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.config.server.TimeoutBudget; + +import java.io.File; + +/** + * A session factory responsible for creating deploy sessions. + * + * @author lulf + * @since 5.1 + */ +public interface SessionFactory { + /** + * Creates a new deployment session from an application package. + * + * + * + * @param applicationDirectory a File pointing to an application. + * @param applicationName name of the application for this new session. + * @param logger a deploy logger where the deploy log will be written. + * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @return a new session + */ + LocalSession createSession(File applicationDirectory, String applicationName, DeployLogger logger, TimeoutBudget timeoutBudget); + + /** + * Creates a new deployment session from an already existing session. + * + * @param existingSession The session to use as base + * @param logger a deploy logger where the deploy log will be written. + * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @return a new session + */ + LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, TimeoutBudget timeoutBudget); +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java new file mode 100644 index 00000000000..3ef6f23b84e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -0,0 +1,170 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; + +import java.io.File; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Serves as the factory of sessions. Takes care of copying files to the correct folder and initializing the + * session state. + * + * @author lulf + * @since 5.1 + */ +public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { + + private static final Logger log = Logger.getLogger(SessionFactoryImpl.class.getName()); + private static final long nonExistingActiveSession = 0; + + private final SessionPreparer sessionPreparer; + private final Curator curator; + private final ConfigCurator configCurator; + private final SessionCounter sessionCounter; + private final ApplicationRepo applicationRepo; + private final Path sessionsPath; + private final TenantFileSystemDirs tenantFileSystemDirs; + private final HostValidator<ApplicationId> hostRegistry; + private final SuperModelGenerationCounter superModelGenerationCounter; + private final ConfigDefinitionRepo defRepo; + private final TenantName tenant; + private final String serverId; + + public SessionFactoryImpl(GlobalComponentRegistry globalComponentRegistry, + SessionCounter sessionCounter, + Path sessionsPath, + ApplicationRepo applicationRepo, + TenantFileSystemDirs tenantFileSystemDirs, HostValidator<ApplicationId> hostRegistry, TenantName tenant) { + this.hostRegistry = hostRegistry; + this.tenant = tenant; + this.sessionPreparer = globalComponentRegistry.getSessionPreparer(); + this.curator = globalComponentRegistry.getCurator(); + this.configCurator = globalComponentRegistry.getConfigCurator(); + this.sessionCounter = sessionCounter; + this.sessionsPath = sessionsPath; + this.applicationRepo = applicationRepo; + this.tenantFileSystemDirs = tenantFileSystemDirs; + this.superModelGenerationCounter = globalComponentRegistry.getSuperModelGenerationCounter(); + this.defRepo = globalComponentRegistry.getConfigDefinitionRepo(); + this.serverId = globalComponentRegistry.getConfigserverConfig().serverId(); + } + + @Override + public LocalSession createSession(File applicationFile, String applicationName, DeployLogger logger, TimeoutBudget timeoutBudget) { + return create(applicationFile, applicationName, logger, nonExistingActiveSession, timeoutBudget); + } + + private void ensureZKPathDoesNotExist(Path sessionPath) { + if (configCurator.exists(sessionPath.getAbsolute())) { + throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); + } + } + + private ApplicationPackage createApplication(File userDir, + File configApplicationDir, + String applicationName, + long sessionId, + long currentlyActiveSession) { + long deployTimestamp = System.currentTimeMillis(); + String user = System.getenv("USER"); + if (user == null) { + user = "unknown"; + } + DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationName, deployTimestamp, sessionId, currentlyActiveSession); + return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); + } + + private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, + long sessionId, + SessionZooKeeperClient sessionZKClient, TimeoutBudget timeoutBudget) throws Exception { + log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); + sessionZKClient.createNewSession(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Creating upload waiter for session " + sessionId); + Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); + log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Done creating upload waiter for session " + sessionId); + LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, new SessionContext(applicationPackage, sessionZKClient, getSessionAppDir(sessionId), applicationRepo, hostRegistry, superModelGenerationCounter)); + log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); + waiter.awaitCompletion(timeoutBudget.timeLeft()); + log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Done waiting on upload waiter for session " + sessionId); + return session; + } + + @Override + public LocalSession createSessionFromExisting(LocalSession existingSession, + DeployLogger logger, + TimeoutBudget timeoutBudget) { + File existingApp = getSessionAppDir(existingSession.getSessionId()); + ApplicationMetaData metaData = FilesApplicationPackage.readMetaData(existingApp); + final ApplicationId existingSessionId = existingSession.getApplicationId(); + + + final long liveApp = getLiveApp(existingSessionId); + logger.log(LogLevel.DEBUG, "Create from existing application id " + existingSessionId + ", live app for it is " + liveApp); + LocalSession session = create(existingApp, metaData.getApplicationName(), logger, liveApp, timeoutBudget); + session.setApplicationId(existingSessionId); + return session; + } + + private LocalSession create(File applicationFile, String applicationName, DeployLogger logger, long currentlyActiveSession, TimeoutBudget timeoutBudget) { + long sessionId = sessionCounter.nextSessionId(); + Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); + log.log(LogLevel.DEBUG, Tenants.logPre(tenant) + "Next session id is " + sessionId + " , sessionIdPath=" + sessionIdPath.getAbsolute()); + try { + ensureZKPathDoesNotExist(sessionIdPath); + SessionZooKeeperClient sessionZooKeeperClient = new SessionZooKeeperClient(curator, configCurator, sessionIdPath, defRepo, serverId); + File userApplicationDir = tenantFileSystemDirs.getUserApplicationDir(sessionId); + IOUtils.copyDirectory(applicationFile, userApplicationDir); + ApplicationPackage applicationPackage = createApplication(applicationFile, userApplicationDir, applicationName, sessionId, currentlyActiveSession); + applicationPackage.writeMetaData(); + logger.log(LogLevel.SPAM, "Application package is written to disk"); + return createSessionFromApplication(applicationPackage, sessionId, sessionZooKeeperClient, timeoutBudget); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("Error creating session: " + e.getMessage(), e); + } + } + + private File getSessionAppDir(long sessionId) { + File appDir = tenantFileSystemDirs.getUserApplicationDir(sessionId); + if (!appDir.exists() || !appDir.isDirectory()) { + throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); + } + return appDir; + } + + @Override + public LocalSession loadSession(long sessionId) { + File sessionDir = getSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); + SessionZooKeeperClient sessionZKClient = new SessionZooKeeperClient(curator, configCurator, sessionIdPath, defRepo, serverId); + SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, hostRegistry, superModelGenerationCounter); + return new LocalSession(tenant, sessionId, sessionPreparer, context); + } + + private long getLiveApp(ApplicationId applicationId) { + List<ApplicationId> applicationIds = applicationRepo.listApplications(); + if (applicationIds.contains(applicationId)) { + return applicationRepo.getSessionIdForApplication(applicationId); + } + return nonExistingActiveSession; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java new file mode 100644 index 00000000000..4057d010b15 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -0,0 +1,283 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.collect.ImmutableList; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.provision.*; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.RotationsCache; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.ModelContextImpl; +import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.http.InvalidApplicationException; +import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; + +import com.yahoo.vespa.curator.Curator; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +/** + * A SessionPreparer is responsible for preparing a session given an application package. + * + * @author lulf + * @since 5.1 + */ +public class SessionPreparer { + + private static final Logger log = Logger.getLogger(SessionPreparer.class.getName()); + + private final ModelFactoryRegistry modelFactoryRegistry; + private final FileDistributionFactory fileDistributionFactory; + private final HostProvisionerProvider hostProvisionerProvider; + private final PermanentApplicationPackage permanentApplicationPackage; + private final ConfigserverConfig configserverConfig; + private final ConfigDefinitionRepo configDefinitionRepo; + private final Curator curator; + private final Zone zone; + + public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry, + FileDistributionFactory fileDistributionFactory, + HostProvisionerProvider hostProvisionerProvider, + PermanentApplicationPackage permanentApplicationPackage, + ConfigserverConfig configserverConfig, + ConfigDefinitionRepo configDefinitionRepo, + Curator curator, + Zone zone) { + this.modelFactoryRegistry = modelFactoryRegistry; + this.fileDistributionFactory = fileDistributionFactory; + this.hostProvisionerProvider = hostProvisionerProvider; + this.permanentApplicationPackage = permanentApplicationPackage; + this.configserverConfig = configserverConfig; + this.configDefinitionRepo = configDefinitionRepo; + this.curator = curator; + this.zone = zone; + } + + /** + * Prepares a session (validates, builds model, writes to zookeeper and distributes files) + * + * @param context Contains classes needed to read/write session data. + * @param logger For storing logs returned in response to client. + * @param params parameters controlling behaviour of prepare. + * @param currentActiveApplicationSet Set of currently active applications. + * @param tenantPath Zookeeper path for the tenant for this session + * @return The config change actions that must be done to handle the activation of the models prepared. + */ + public ConfigChangeActions prepare(SessionContext context, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath) + { + Preparation prep = new Preparation(context, logger, params, currentActiveApplicationSet, tenantPath); + prep.preprocess(); + try { + prep.buildModels(); + prep.makeResult(); + if (!params.isDryRun()) { + prep.writeStateZK(); + prep.writeRotZK(); + prep.distribute(); + prep.reloadDeployFileDistributor(); + } + return prep.result(); + } catch (IllegalArgumentException e) { + throw new InvalidApplicationException("Invalid application package", e); + } + } + + private class Preparation { + + final SessionContext context; + final DeployLogger logger; + final PrepareParams params; + + final Optional<ApplicationSet> currentActiveApplicationSet; + final Path tenantPath; + final ApplicationId applicationId; + final RotationsCache rotationsCache; + final Set<Rotation> rotations; + final ModelContext.Properties properties; + + private ApplicationPackage applicationPackage; + private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; + private PrepareResult prepareResult; + + private final PreparedModelsBuilder preparedModelsBuilder; + + Preparation(SessionContext context, DeployLogger logger, PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath) { + this.context = context; + this.logger = logger; + this.params = params; + this.currentActiveApplicationSet = currentActiveApplicationSet; + this.tenantPath = tenantPath; + + this.applicationId = params.getApplicationId(); + this.rotationsCache = new RotationsCache(curator, tenantPath); + this.rotations = getRotations(params.rotations()); + this.properties = new ModelContextImpl.Properties(params.getApplicationId(), + configserverConfig.multitenant(), + ConfigServerSpec.fromConfig(configserverConfig), + configserverConfig.hostedVespa(), + zone, + rotations); + this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, + permanentApplicationPackage, + configserverConfig, + configDefinitionRepo, + curator, + zone, + fileDistributionFactory, + hostProvisionerProvider, + context, + logger, + params, + currentActiveApplicationSet, + tenantPath); + } + + void checkTimeout(String step) { + if (! params.getTimeoutBudget().hasTimeLeft()) { + String used = params.getTimeoutBudget().timesUsed(); + throw new RuntimeException("prepare timed out "+used+" after "+step+" step: " + applicationId); + } + } + + void preprocess() { + try { + this.applicationPackage = context.getApplicationPackage().preprocess( + properties.zone(), + null, + logger); + } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) { + throw new RuntimeException("Error deploying application package", e); + } + checkTimeout("preprocess"); + } + + void buildModels() { + this.modelResultList = preparedModelsBuilder.buildModels(applicationId, applicationPackage); + checkTimeout("build models"); + } + + void makeResult() { + this.prepareResult = new PrepareResult(modelResultList); + checkTimeout("making result from models"); + } + + void writeStateZK() { + log.log(LogLevel.DEBUG, "Writing application package state to zookeeper"); + writeStateToZooKeeper(context.getSessionZooKeeperClient(), applicationPackage, params, logger, + prepareResult.getFileRegistries(), prepareResult.getProvisionInfos()); + checkTimeout("write state to zookeeper"); + } + + void writeRotZK() { + rotationsCache.writeRotationsToZooKeeper(applicationId, rotations); + checkTimeout("write rotations to zookeeper"); + } + + void distribute() { + prepareResult.asList().forEach(modelResult -> modelResult.model + .distributeFiles(modelResult.fileDistributionProvider.getFileDistribution())); + checkTimeout("distribute files"); + } + + void reloadDeployFileDistributor() { + if (prepareResult.asList().isEmpty()) return; + PreparedModelsBuilder.PreparedModelResult aModelResult = prepareResult.asList().get(0); + aModelResult.model.reloadDeployFileDistributor(aModelResult.fileDistributionProvider.getFileDistribution()); + checkTimeout("reload all deployed files in file distributor"); + } + + ConfigChangeActions result() { + return prepareResult.getConfigChangeActions(); + } + + private Set<Rotation> getRotations(Set<Rotation> rotations) { + if (rotations == null || rotations.isEmpty()) { + rotations = rotationsCache.readRotationsFromZooKeeper(applicationId); + } + return rotations; + } + + } + + private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, + ApplicationPackage applicationPackage, + PrepareParams prepareParams, + DeployLogger deployLogger, + Map<Version, FileRegistry> fileRegistryMap, + Map<Version, ProvisionInfo> provisionInfoMap) { + ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); + try { + zkDeployer.deploy(applicationPackage, fileRegistryMap, provisionInfoMap); + zooKeeperClient.writeApplicationId(prepareParams.getApplicationId()); + } catch (RuntimeException | IOException e) { + zkDeployer.cleanup(); + throw new RuntimeException("Error preparing session", e); + } + } + + /** The result of preparation over all model versions */ + private static class PrepareResult { + + private final ImmutableList<PreparedModelsBuilder.PreparedModelResult> results; + + public PrepareResult(List<PreparedModelsBuilder.PreparedModelResult> results) { + this.results = ImmutableList.copyOf(results); + } + + /** Returns the results for each model as an immutable list */ + public List<PreparedModelsBuilder.PreparedModelResult> asList() { return results; } + + public Map<Version, ProvisionInfo> getProvisionInfos() { + return results.stream() + .filter(result -> result.model.getProvisionInfo().isPresent()) + .collect(Collectors.toMap((prepareResult -> prepareResult.version), + (prepareResult -> prepareResult.model.getProvisionInfo().get()))); + } + + public Map<Version, FileRegistry> getFileRegistries() { + return results.stream() + .collect(Collectors.toMap((prepareResult -> prepareResult.version), + (prepareResult -> prepareResult.fileDistributionProvider.getFileRegistry()))); + } + + /** + * Collects the config change actions from all model factory creations and returns the aggregated union of these actions. + * A system in the process of upgrading Vespa will have hosts running both version X and Y, and this will change + * during the upgrade process. Trying to be smart about which actions to perform on which hosts depending + * on the version running will be a nightmare to maintain. A pragmatic approach is therefore to just use the + * union of all actions as this will give the correct end result at the cost of perhaps restarting nodes twice + * (once for the upgrading case and once for a potential restart action). + */ + public ConfigChangeActions getConfigChangeActions() { + return new ConfigChangeActions(results.stream(). + map(result -> result.actions). + flatMap(actions -> actions.stream()). + collect(Collectors.toList())); + } + + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepo.java new file mode 100644 index 00000000000..872e6117637 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepo.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.http.NotFoundException; + +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; + +/** + * A generic session repository that can store any type of session that extends the abstract interface. + * + * @author lulf + * @since 5.1 + */ +public class SessionRepo<SESSIONTYPE extends Session> { + + private final HashMap<Long, SESSIONTYPE> sessions = new HashMap<>(); + + public synchronized void addSession(SESSIONTYPE session) { + final long sessionId = session.getSessionId(); + if (sessions.containsKey(sessionId)) { + throw new IllegalArgumentException("There already exists a session with id '" + sessionId + "'"); + } + sessions.put(sessionId, session); + } + + public synchronized void removeSession(long id) { + if ( ! sessions.containsKey(id)) { + throw new IllegalArgumentException("No such session exists '" + id + "'"); + } + sessions.remove(id); + } + + /** + * Gets a Session + * + * @param id session id + * @return a session belonging to the id supplied, or null if no session with the id was found + */ + public synchronized SESSIONTYPE getSession(long id) { + return sessions.get(id); + } + + /** + * Gets a Session with a timeout + * + * @param id session id + * @param timeoutInMillis timeout for getting session (loops and wait for session to show up if not found) + * @return a session belonging to the id supplied, or null if no session with the id was found + */ + public synchronized SESSIONTYPE getSession(long id, long timeoutInMillis) { + try { + return internalGetSession(id, timeoutInMillis); + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted while retrieving session with id " + id); + } + } + + private synchronized SESSIONTYPE internalGetSession(long id, long timeoutInMillis) throws InterruptedException { + TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofMillis(timeoutInMillis)); + do { + SESSIONTYPE session = getSession(id); + if (session != null) { + return session; + } + wait(100); + } while (timeoutBudget.hasTimeLeft()); + throw new NotFoundException("Unable to retrieve session with id " + id + " before timeout was reached"); + } + + public synchronized Collection<SESSIONTYPE> listSessions() { + return new ArrayList<>(sessions.values()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java new file mode 100644 index 00000000000..37dff639a35 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.curator.Curator; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.NodeCacheListener; + +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +/** + * Watches one particular session (/vespa/config/apps/n/sessionState in ZK) + * The session must be in the session repo. + * + * @author vegardh + */ +public class SessionStateWatcher implements NodeCacheListener { + + private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); + private final Curator.FileCache fileCache; + private final ReloadHandler reloadHandler; + private final RemoteSession session; + private final MetricUpdater metrics; + private final Executor executor; + + public SessionStateWatcher(Curator.FileCache fileCache, ReloadHandler reloadHandler, RemoteSession session, MetricUpdater metrics) throws Exception { + executor = Executors.newSingleThreadExecutor(ThreadFactoryFactory.getThreadFactory(SessionStateWatcher.class.getName() + "-" + session)); + this.fileCache = fileCache; + this.reloadHandler = reloadHandler; + this.session = session; + this.metrics = metrics; + this.fileCache.start(); + this.fileCache.addListener(this); + } + + private void sessionChanged(Session.Status status) { + log.log(LogLevel.DEBUG, session.logPre()+"Session change: Session " + session.getSessionId() + " changed status to " + status); + + // valid for NEW -> PREPARE transitions, not ACTIVATE -> PREPARE. + if (status.equals(Session.Status.PREPARE)) { + log.log(LogLevel.DEBUG, session.logPre() + "Loading prepared session: " + session.getSessionId()); + session.loadPrepared(); + } else if (status.equals(Session.Status.ACTIVATE)) { + session.makeActive(reloadHandler); + } else if (status.equals(Session.Status.DEACTIVATE)) { + session.deactivate(); + } + } + + public long getSessionId() { + return session.getSessionId(); + } + + public void close() { + try { + fileCache.close(); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Exception when closing watcher", e); + } + } + + @Override + public void nodeChanged() throws Exception { + executor.execute(() -> { + try { + ChildData data = fileCache.getCurrentData(); + if (data != null) { + sessionChanged(Session.Status.parse(Utf8.toString(fileCache.getCurrentData().getData()))); + } + } catch (Exception e) { + log.log(LogLevel.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); + metrics.incSessionChangeErrors(); + } + }); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java new file mode 100644 index 00000000000..e17667c2a5a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -0,0 +1,213 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.TenantName; +import com.yahoo.transaction.Transaction; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.StaticConfigDefinitionRepo; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; +import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +import java.util.concurrent.TimeUnit; + +/** + * Zookeeper client for a specific session. Can be used to read and write session status + * and create and get prepare and active barrier. + * + * @author lulf + * @since 5.1 + */ +public class SessionZooKeeperClient { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SessionZooKeeperClient.class.getName()); + static final String APPLICATION_ID_PATH = "applicationId"; + static final String CREATE_TIME_PATH = "createTime"; + private final Curator curator; + private final ConfigCurator configCurator; + private final Path rootPath; + private final Path sessionStatusPath; + private final String serverId; + private final ServerCacheLoader cacheLoader; + + // Only for testing when cache loader does not need cache entries. + public SessionZooKeeperClient(Curator curator, Path rootPath) { + this(curator, ConfigCurator.create(curator), rootPath, new StaticConfigDefinitionRepo(), ""); + } + + public SessionZooKeeperClient(Curator curator, ConfigCurator configCurator, Path rootPath, ConfigDefinitionRepo definitionRepo, String serverId) { + this.curator = curator; + this.configCurator = configCurator; + this.rootPath = rootPath; + this.serverId = serverId; + this.sessionStatusPath = rootPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + this.cacheLoader = new ServerCacheLoader(configCurator, rootPath, definitionRepo); + } + + public void writeStatus(Session.Status sessionStatus) { + try { + createWriteStatusTransaction(sessionStatus).commit(); + } catch (Exception e) { + throw new RuntimeException("Unable to write session status", e); + } + } + + public Session.Status readStatus() { + try { + String data = configCurator.getData(sessionStatusPath.getAbsolute()); + return Session.Status.parse(data); + } catch (Exception e) { + log.log(LogLevel.INFO, "Unable to read session status, assuming it was deleted"); + return Session.Status.NONE; + } + } + + Curator.CompletionWaiter createPrepareWaiter() { + return createCompletionWaiter(PREPARE_BARRIER); + } + + Curator.CompletionWaiter createActiveWaiter() { + return createCompletionWaiter(ACTIVE_BARRIER); + } + + Curator.CompletionWaiter getPrepareWaiter() { + return getCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); + } + + Curator.CompletionWaiter getActiveWaiter() { + return getCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); + } + + Curator.CompletionWaiter getUploadWaiter() { + return getCompletionWaiter(getWaiterPath(UPLOAD_BARRIER)); + } + + private static final String PREPARE_BARRIER = "prepareBarrier"; + private static final String ACTIVE_BARRIER = "activeBarrier"; + private static final String UPLOAD_BARRIER = "uploadBarrier"; + + private Path getWaiterPath(String barrierName) { + return rootPath.append(barrierName); + } + + private int getNumberOfMembers() { + /* + * The number of members required in a barrier is the majority of servers. + */ + int numServers = curator.serverCount(); + return (numServers / 2) + 1; + } + + private Curator.CompletionWaiter createCompletionWaiter(String waiterNode) { + return curator.createCompletionWaiter(rootPath, waiterNode, getNumberOfMembers(), serverId); + } + + private Curator.CompletionWaiter getCompletionWaiter(Path path) { + return curator.getCompletionWaiter(path, getNumberOfMembers(), serverId); + } + + public void delete() { + try { + log.log(LogLevel.DEBUG, "Deleting " + rootPath.getAbsolute()); + configCurator.deleteRecurse(rootPath.getAbsolute()); + } catch (RuntimeException e) { + log.log(LogLevel.INFO, "Error deleting session (" + rootPath.getAbsolute() + ") from zookeeper"); + } + } + + public ApplicationPackage loadApplicationPackage() { + return new ZKApplicationPackage(configCurator, rootPath); + } + + public ServerCache loadServerCache() { + return cacheLoader.loadCache(); + } + + public void writeApplicationId(ApplicationId id) { + String path = getApplicationIdPath(); + try { + configCurator.putData(path, id.serializedForm()); + } catch (RuntimeException e) { + throw new RuntimeException("Unable to write application id '" + id + "' to '" + path + "'", e); + } + } + + private String getApplicationIdPath() { + return rootPath.append(APPLICATION_ID_PATH).getAbsolute(); + } + + public ApplicationId readApplicationId(TenantName tenant) { + String path = getApplicationIdPath(); + try { + // Fallback for cases where id never existed. + if ( ! configCurator.exists(path)) { + // TODO: DEBUG LOG + log.log(LogLevel.INFO, "Unable to locate application id at '" + path + "', returning default"); + return ApplicationId.defaultId(); + } + return ApplicationId.fromSerializedForm(tenant, configCurator.getData(path)); + } catch (RuntimeException e) { + throw new RuntimeException("Unable to read application id from '" + path + "'", e); + } + } + + // in seconds + public long readCreateTime() { + String path = getCreateTimePath(); + if (!configCurator.exists(path)) return 0l; + return Long.parseLong(configCurator.getData(path)); + } + + private String getCreateTimePath() { + return rootPath.append(CREATE_TIME_PATH).getAbsolute(); + } + + ProvisionInfo getProvisionInfo() { + return loadApplicationPackage().getProvisionInfoMap().values().stream() + .reduce((infoA, infoB) -> infoA.merge(infoB)) + .orElseThrow(() -> new IllegalStateException("Trying to read provision info, but no provision info exists")); + } + + public ZooKeeperDeployer createDeployer(DeployLogger logger) { + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, logger, true, rootPath); + return new ZooKeeperDeployer(zkClient); + } + + public Transaction createWriteStatusTransaction(Session.Status status) { + String path = sessionStatusPath.getAbsolute(); + CuratorTransaction transaction = new CuratorTransaction(curator); + if (configCurator.exists(path)) { + transaction.add(CuratorOperations.setData(sessionStatusPath.getAbsolute(), Utf8.toBytes(status.name()))); + } else { + transaction.add(CuratorOperations.create(sessionStatusPath.getAbsolute(), Utf8.toBytes(status.name()))); + } + return transaction; + } + + /** + * Create necessary paths atomically for a new session. + * @param createTime Time of session creation. + * @param timeUnit Time unit of createTime. + */ + public void createNewSession(long createTime, TimeUnit timeUnit) { + CuratorTransaction transaction = new CuratorTransaction(curator); + transaction.add(CuratorOperations.create(rootPath.getAbsolute())); + transaction.add(CuratorOperations.create(rootPath.append(UPLOAD_BARRIER).getAbsolute())); + transaction.add(createWriteStatusTransaction(Session.Status.NEW).operations()); + transaction.add(CuratorOperations.create(getCreateTimePath(), Utf8.toBytes(String.valueOf(timeUnit.toSeconds(createTime))))); + transaction.commit(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SilentDeployLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SilentDeployLogger.java new file mode 100644 index 00000000000..9121eab8d2f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SilentDeployLogger.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.config.application.api.DeployLogger; + +/** + * The purpose of this is to mute the log messages from model and application building in {@link RemoteSession} that is triggered by {@link SessionStateWatcher}, since those messages already + * have been emitted by the prepare handler, for the same prepare operation. + * + * @author vegardh + * + */ +public class SilentDeployLogger implements DeployLogger { + + private static final Logger log = Logger.getLogger(SilentDeployLogger.class.getName()); + + @Override + public void log(Level level, String message) { + log.log(Level.FINE, message); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java new file mode 100644 index 00000000000..2a09e46d821 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.version; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; + +/** + * Contains version information for this configserver. + * + * @author lulf + */ +public class VersionState { + + private final File versionFile; + + @Inject + public VersionState(ConfigserverConfig config) { + this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version")); + } + + public VersionState(File versionFile) { + this.versionFile = versionFile; + } + + public boolean isUpgraded() { + return currentVersion().compareTo(storedVersion()) > 0; + } + + public void saveNewVersion() { + try (FileWriter writer = new FileWriter(versionFile)) { + writer.write(currentVersion().toSerializedForm()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Version storedVersion() { + try (FileReader reader = new FileReader(versionFile)) { + return Version.fromString(IOUtils.readAll(reader)); + } catch (Exception e) { + return Version.fromIntValues(0, 0, 0); // Use an old value to signal we don't know + } + } + + public Version currentVersion() { + return Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); + } + + @Override + public String toString() { + return String.format("Current version:%s, stored version:%s", currentVersion(), storedVersion()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java new file mode 100644 index 00000000000..af8d2fed95c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -0,0 +1,429 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.zookeeper.ZooKeeperServer; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A (stateful) curator wrapper for the config server. This simplifies Curator method calls used by the config server + * and knows about how config content is mapped to node names and stored. + * <p> + * Usage details: + * Config ids are stored as foo#bar#c0 instead of foo/bar/c0, for simplicity. + * Keep the amount of domain-specific logic here to a minimum. + * Data for one application x is stored on this form: + * /vespa/config/apps/x/defconfigs + * /vespa/config/apps/x/userapp + * The different types of configs are stored on this form (ie. names of the ZK nodes under their respective + * paths): + * <p> + * Def configs are stored on the form name,version + * The user application structure is exactly the same as in the user's app dir during deploy. + * The current live app id (for example x) is stored in the node /vespa/config/liveapp + * It is updated outside this class, typically in config server during reload-config. + * Some methods have retries and/or reconnect. This is necessary because ZK will throw on certain scenarios, + * even though it will recover from it itself, @see http://wiki.apache.org/hadoop/ZooKeeper/ErrorHandling + * + * @author vegardh + * @author bratseth + * @since 5.0 + */ +public class ConfigCurator { + + /** Path for def files, under one app */ + public static final String DEFCONFIGS_ZK_SUBPATH = "/defconfigs"; + + /** Path for def files, under one app */ + public static final String USER_DEFCONFIGS_ZK_SUBPATH = "/userdefconfigs"; + + /** Path for metadata about an application */ + public static final String META_ZK_PATH = "/meta"; + + /** Path for the app package's dir structure, under one app */ + public static final String USERAPP_ZK_SUBPATH = "/userapp"; + + /** Path for session state */ + public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState"; + + protected static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; + + private final Curator curator; + + public static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigCurator.class.getName()); + + /** The number of zookeeper operations done with this ZKFacade instance */ + private final AtomicInteger operations = new AtomicInteger(); + + /** The number of zookeeper read operations done with this ZKFacade instance */ + private final AtomicInteger readOperations = new AtomicInteger(); + + /** The number of zookeeper write operations done with this ZKFacade instance */ + private final AtomicInteger writeOperations = new AtomicInteger(); + + /** The maximum size of a ZooKeeper node */ + private final int maxNodeSize; + + /** + * Sets up thread local zk access if not done before and returns a facade object + * + * @return a ZKFacade object + */ + public static ConfigCurator create(Curator curator, int juteMaxBuffer) { + return new ConfigCurator(curator, juteMaxBuffer); + } + + public static ConfigCurator create(Curator curator) { + return new ConfigCurator(curator, 1024*1024*10); + } + + @Inject + public ConfigCurator(Curator curator, ZooKeeperServer server) { + this(curator, server.getConfig().juteMaxBuffer()); + } + + private ConfigCurator(Curator curator, int maxNodeSize) { + this.curator = curator; + this.maxNodeSize = maxNodeSize; + log.log(LogLevel.CONFIG, "Using jute max buffer size " + this.maxNodeSize); + testZkConnection(); + } + + /** Returns the curator instance this wraps */ + public Curator curator() { return curator; } + + /** Cleans and creates a zookeeper completely */ + public void initAndClear(String path) { + try { + operations.incrementAndGet(); + readOperations.incrementAndGet(); + if (exists(path)) + deleteRecurse(path); + createRecurse(path); + } + catch (Exception e) { + throw new RuntimeException("Exception clearing path " + path + " in ZooKeeper", e); + } + } + + /** Creates a path. If the path already exists this does nothing. */ + private void createRecurse(String path) { + try { + if (exists(path)) return; + curator.framework().create().creatingParentsIfNeeded().forPath(path); + } + catch (Exception e) { + throw new RuntimeException("Exception creating path " + path + " in ZooKeeper", e); + } + } + + /** Returns the data at a path and node. Replaces / by # in node names. Returns null if the path doesn't exist. */ + public String getData(String path, String node) { + return getData(createFullPath(path, node)); + } + + /** Returns the data at a path. Returns null if the path doesn't exist. */ + public String getData(String path) { + byte[] data = getBytes(path); + return (data == null) ? null : Utf8.toString(data); + } + + /** Returns the data at a path and node. Replaces / by # in node names. Returns null if the path doesn't exist. */ + public byte[] getBytes(String path, String node) { + return getBytes(createFullPath(path, node)); + } + + /** + * Returns the data at a path, or null if the path does not exist. + * + * @param path a String with a pathname. + * @return a byte array with data. + */ + public byte[] getBytes(String path) { + try { + if ( ! exists(path)) return null; // TODO: Ugh + operations.incrementAndGet(); + readOperations.incrementAndGet(); + return curator.framework().getData().forPath(path); + } + catch (Exception e) { + throw new RuntimeException("Exception reading from path " + path + " in ZooKeeper", e); + } + } + + /** Returns whether a path exists in zookeeper */ + public boolean exists(String path, String node) { + return exists(createFullPath(path, node)); + } + + /** Returns whether a path exists in zookeeper */ + public boolean exists(String path) { + try { + return curator.framework().checkExists().forPath(path) != null; + } + catch (Exception e) { + throw new RuntimeException("Exception checking existence of path " + path + " in ZooKeeper", e); + } + } + + /** Creates a Zookeeper node. If the node already exists this does nothing. */ + public void createNode(String path) { + if ( ! exists(path)) + createRecurse(path); + operations.incrementAndGet(); + writeOperations.incrementAndGet(); + } + + /** Creates a Zookeeper node synchronously. Replaces / by # in node names. */ + public void createNode(String path, String node) { + createNode(createFullPath(path, node)); + } + + private String createFullPath(String path, String node) { + return path + "/" + toConfigserverName(node); + } + + /** Sets data at a given path and name. Replaces / by # in node names. Creates the node if it doesn't exist */ + public void putData(String path, String node, String data) { + putData(path, node, Utf8.toBytes(data)); + } + + /** Sets data at a given path. Creates the node if it doesn't exist */ + public void putData(String path, String data) { + putData(path, Utf8.toBytes(data)); + } + + private void ensureDataIsNotTooLarge(byte[] toPut, String path) { + if (toPut.length >= maxNodeSize) { + throw new IllegalArgumentException("Error: too much zookeeper data in node: " + + "[" + toPut.length + " bytes] (path " + path + ")"); + } + } + + /** Sets data at a given path and name. Replaces / by # in node names. Creates the node if it doesn't exist */ + public void putData(String path, String node, byte[] data) { + putData(createFullPath(path, node), data); + } + + /** Sets data at a given path. Creates the path if it doesn't exist */ + public void putData(String path, byte[] data) { + try { + ensureDataIsNotTooLarge(data, path); + operations.incrementAndGet(); + writeOperations.incrementAndGet(); + if (exists(path)) + curator.framework().setData().forPath(path, data); + else + curator.framework().create().creatingParentsIfNeeded().forPath(path, data); + } + catch (Exception e) { + throw new RuntimeException("Exception writing to path " + path + " in ZooKeeper", e); + } + } + + /** Sets data at an existing node. Replaces / by # in node names. */ + public void setData(String path, String node, String data) { + setData(path, node, Utf8.toBytes(data)); + } + + /** Sets data at an existing node. Replaces / by # in node names. */ + public void setData(String path, String node, byte[] data) { + setData(createFullPath(path, node), data); + } + + /** Sets data at an existing node. Replaces / by # in node names. */ + public void setData(String path, byte[] data) { + try { + ensureDataIsNotTooLarge(data, path); + operations.incrementAndGet(); + writeOperations.incrementAndGet(); + curator.framework().setData().forPath(path, data); + } + catch (Exception e) { + throw new RuntimeException("Exception writing to path " + path + " in ZooKeeper", e); + } + } + + /** + * Replaces / with # in the given node. + * + * @param node a zookeeper node name + * @return a config server node name + */ + protected String toConfigserverName(String node) { + if (node.startsWith("/")) node = node.substring(1); + return node.replaceAll("/", "#"); + } + + /** + * Lists thh children at the given path. + * + * @return the local names of the children at this path, or an empty list (never null) if none. + */ + public List<String> getChildren(String path) { + try { + operations.incrementAndGet(); + readOperations.incrementAndGet(); + return curator.framework().getChildren().forPath(path); + } + catch (Exception e) { + throw new RuntimeException("Exception getting children of path " + path + " in ZooKeeper", e); + } + } + + /** + * Puts config definition data and metadata into ZK. + * + * @param name The config definition name (including namespace) + * @param version The config definition version + * @param path /zoopath + * @param data The contents to write to ZK (as a byte array) + */ + public void putDefData(String name, String version, String path, byte[] data) { + if (version == null) { + putData(path, name, data); + } else { + String fullPath = createFullPath(path, name + "," + version); + if (exists(fullPath)) { + // TODO This should not happen when all the compatibility hacks in 5.1 have been removed + log.log(LogLevel.INFO, "There already exists a config definition '" + name + "', skipping feeding this one to ZooKeeper"); + } + else { + putData(fullPath, data); + } + } + } + + /** + * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, + * and dirs called CVS. + * + * @param dir directory which holds the summary class part files + * @param path zookeeper path + * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper + * @param recurse recurse subdirectories + */ + public void feedZooKeeper(File dir, String path, FilenameFilter filenameFilter, boolean recurse) { + try { + if (filenameFilter == null) { + filenameFilter = acceptsAllFileNameFilter; + } + if (!dir.isDirectory()) { + log.fine(dir.getCanonicalPath() + " is not a directory. Not feeding the files into ZooKeeper."); + return; + } + for (File file : listFiles(dir, filenameFilter)) { + if (file.getName().startsWith(".")) continue; //.svn , .git ... + if ("CVS".equals(file.getName())) continue; + if (file.isFile()) { + byte[] contents = IOUtils.readFileBytes(file); + putData(path, file.getName(), contents); + } else if (recurse && file.isDirectory()) { + createNode(path, file.getName()); + feedZooKeeper(file, path + '/' + file.getName(), filenameFilter, recurse); + } + } + } + catch (IOException e) { + throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); + } + } + + /** + * Same as normal listFiles, but use the filter only for normal files + * + * @param dir directory to list files in + * @param filter A FilenameFilter which decides which files in dir are listed + * @return an array of Files + */ + protected File[] listFiles(File dir, FilenameFilter filter) { + File[] rawList = dir.listFiles(); + List<File> ret = new ArrayList<>(); + if (rawList != null) { + for (File f : rawList) { + if (f.isDirectory()) { + ret.add(f); + } else { + if (filter.accept(dir, f.getName())) { + ret.add(f); + } + } + } + } + return ret.toArray(new File[ret.size()]); + } + + /** + * The node string for a given config and path. Ignores id and/or version if the path is that of a + * data set that doesn't use id or version. + * + * @param path a zookeeper path + * @param name config definition name + * @param version config definition version + * @param id config id + * @return a String with path to a zookeeper node for a given config and path + */ + public static String getZkNodePath(String path, String name, String version, String id) { + if (path.endsWith(DEFCONFIGS_ZK_SUBPATH) || path.endsWith(USER_DEFCONFIGS_ZK_SUBPATH)) + return getConfigNodeName(name, version); + throw new IllegalArgumentException("Don't know how data in " + path + " is organised in ZK"); + } + + public static String getConfigNodeName(String name, String version) { + return getConfigNodeName(name, version, null); + } + + public static String getConfigNodeName(String name, String version, String configId) { + if (configId == null || "".equals(configId)) + return name + "," + version; + return name + "," + version + "," + configId; + } + + /** Deletes the node at the given path, and any children it may have. If the node does not exist this does nothing */ + public void deleteRecurse(String path) { + try { + if ( ! exists(path)) return; + curator.framework().delete().deletingChildrenIfNeeded().forPath(path); + } + catch (Exception e) { + throw new RuntimeException("Exception deleting path " + path, e); + } + } + + public Integer getNumberOfOperations() { return operations.intValue(); } + + public Integer getNumberOfReadOperations() { + return readOperations.intValue(); + } + + public Integer getNumberOfWriteOperations() { + return writeOperations.intValue(); + } + + + private void testZkConnection() { // This is not necessary, but allows us to give a useful error message + if (curator.connectionSpec().isEmpty()) return; + try { + curator.framework().checkExists().forPath("/dummy"); + } + catch (Exception e) { + log.log(LogLevel.ERROR, "Unable to contact ZooKeeper on " + curator.connectionSpec() + + ". Please verify for all configserver nodes that " + + "services.addr_configserver points to the correct configserver(s), " + + "the same configserver(s) as in services.xml, and that they are started. " + + "Check the log(s) for configserver errors. Aborting.", e); + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java new file mode 100644 index 00000000000..20572b938ba --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.recipes.CuratorCounter; +import com.yahoo.vespa.curator.Curator; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A counter that sets its initial value to the number of apps in zookeeper if no counter value is set. Subclass + * this to get that behavior. + * + * @author lulf + * @since 5.1 + */ +public class InitializedCounter { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(InitializedCounter.class.getName()); + protected final CuratorCounter counter; + private final String sessionsDirPath; + + public InitializedCounter(Curator curator, String counterPath, String sessionsDirPath) { + this.sessionsDirPath = sessionsDirPath; + this.counter = new CuratorCounter(curator, counterPath); + initializeCounterValue(getLatestSessionId(ConfigCurator.create(curator), sessionsDirPath)); + } + + private void initializeCounterValue(Long latestSessionId) { + log.log(LogLevel.DEBUG, "path=" + sessionsDirPath + ", current=" + latestSessionId); + if (latestSessionId != null) { + counter.initialize(latestSessionId); + } else { + counter.initialize(1); + } + } + + /** + * Checks if an application exists in Zookeeper. + * + * @return true, if an application exists, false otherwise + */ + private static boolean applicationExists(ConfigCurator configCurator, String appsPath) { + // TODO Need to try and catch now since interface should not expose Zookeeper exceptions + try { + return configCurator.exists(appsPath); + } catch (Exception e) { + log.log(LogLevel.WARNING, e.getMessage()); + return false; + } + } + + /** + * Returns the application generation for the most recently deployed application from ZK, + * or null if no application has been deployed yet + * + * @return generation of the latest deployed application + */ + private static Long getLatestSessionId(ConfigCurator configCurator, String appsPath) { + if (!applicationExists(configCurator, appsPath)) return null; + Long newestGeneration = null; + try { + if (!getDeployedApplicationGenerations(configCurator, appsPath).isEmpty()) { + newestGeneration = Collections.max(getDeployedApplicationGenerations(configCurator, appsPath)); + } + } catch (Exception e) { + log.log(LogLevel.WARNING, "Could not get newest application generation from Zookeeper"); + } + return newestGeneration; + } + + private static List<Long> getDeployedApplicationGenerations(ConfigCurator configCurator, String appsPath) { + ArrayList<Long> generations = new ArrayList<>(); + try { + List<String> stringGenerations = configCurator.getChildren(appsPath); + if (stringGenerations != null && !(stringGenerations.isEmpty())) { + for (String s : stringGenerations) { + generations.add(Long.parseLong(s)); + } + } + } catch (RuntimeException e) { + log.log(LogLevel.WARNING, "Could not get application generations from Zookeeper"); + } + return generations; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java new file mode 100644 index 00000000000..cad164f8614 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/SessionCounter.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; + +/** + * A counter keeping track of session ids in an atomic fashion across multiple config servers. + * + * @author lulf + * @since 5.1 + */ +public class SessionCounter extends InitializedCounter { + + public SessionCounter(Curator curator, Path rootPath, Path sessionsDir) { + super(curator, rootPath.append("sessionCounter").getAbsolute(), sessionsDir.getAbsolute()); + } + + /** + * Atomically increment and return next session id. + * + * @return a new session id. + */ + public long nextSessionId() { + return counter.next(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java new file mode 100644 index 00000000000..4aedb487352 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.path.Path; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author lulf + * @author vegardh + * @since 5.1 + */ +class ZKApplicationFile extends ApplicationFile { + private static final Logger log = Logger.getLogger("ZKApplicationFile"); + private final ZKLiveApp zkApp; + private final ObjectMapper mapper = new ObjectMapper(); + + public ZKApplicationFile(Path path, ZKLiveApp app) { + super(path); + this.zkApp = app; + } + + @Override + public boolean isDirectory() { + String zkPath = getZKPath(path); + if (zkApp.exists(zkPath)) { + String data = zkApp.getData(zkPath); + if (data == null || data.isEmpty() || !zkApp.getChildren(zkPath).isEmpty()) { + return true; + } + } + return false; + } + + @Override + public boolean exists() { + try { + String zkPath = getZKPath(path); + return zkApp.exists(zkPath); + } catch (RuntimeException e) { + return false; + } + } + + @Override + public ApplicationFile delete() { + if (!listFiles().isEmpty()) { + throw new RuntimeException("Can't delete, directory not empty: " + this); + } + zkApp.deleteRecurse(getZKPath(path)); + writeMetaFile(null, ContentStatusDeleted); + return this; + } + + @Override + public Reader createReader() throws FileNotFoundException { + String zkPath = getZKPath(path); + String data = zkApp.getData(zkPath); + if (data == null) { + throw new FileNotFoundException("No such path: " + path); + } + return new StringReader(data); + } + + @Override + public InputStream createInputStream() throws FileNotFoundException { + String zkPath = getZKPath(path); + byte[] data = zkApp.getBytes(zkPath); + if (data == null) { + throw new FileNotFoundException("No such path: " + path); + } + return new ByteArrayInputStream(data); + } + + @Override + public ApplicationFile createDirectory() { + String zkPath = getZKPath(path); + if (isDirectory()) return this; + if (exists()) { + throw new IllegalArgumentException("Unable to create directory, file exists: " + path); + } + zkApp.create(zkPath); + writeMetaFile(null, ContentStatusNew); + return this; + } + + @Override + public ApplicationFile writeFile(Reader input) { + // foo/bar/baz.txt + String zkPath = getZKPath(path); + try { + String data = IOUtils.readAll(input); + String status = ContentStatusNew; + if (zkApp.exists(zkPath)) { + status = ContentStatusChanged; + } + zkApp.putData(zkPath, data); + writeMetaFile(data, status); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public List<ApplicationFile> listFiles(PathFilter filter) { + String userPath = getZKPath(path); + List<ApplicationFile> ret = new ArrayList<>(); + for (String zkChild : zkApp.getChildren(userPath)) { + Path childPath = path.append(zkChild); + // Ignore dot-files. + if (!childPath.getName().startsWith(".") && filter.accept(childPath)) { + ret.add(new ZKApplicationFile(childPath, zkApp)); + } + } + return ret; + } + + private static String getZKPath(Path path) { + if (path.isRoot()) { + return ConfigCurator.USERAPP_ZK_SUBPATH; + } + return ConfigCurator.USERAPP_ZK_SUBPATH + "/" + path.getRelative(); + } + + private void writeMetaFile(String input, String status) { + String metaPath = getZKPath(getMetaPath()); + StringWriter writer = new StringWriter(); + try { + mapper.writeValue(writer, new MetaData(status, input == null ? "" : ConfigUtils.getMd5(input))); + log.log(LogLevel.DEBUG, "Writing meta file to " + metaPath); + zkApp.putData(metaPath, writer.toString()); + } catch (IOException e) { + throw new RuntimeException("Error writing meta file to " + metaPath, e); + } + } + + public MetaData getMetaData() { + String metaPath = getZKPath(getMetaPath()); + log.log(LogLevel.DEBUG, "Getting metadata for " + metaPath); + if (!zkApp.exists(getZKPath(path))) { + if (zkApp.exists(metaPath)) { + return getMetaDataFromZk(metaPath); + } else { + return null; + } + } + if (zkApp.exists(metaPath)) { + return getMetaDataFromZk(metaPath); + } + return new MetaData(ContentStatusNew, isDirectory() ? "" : ConfigUtils.getMd5(zkApp.getData(getZKPath(path)))); + } + + private MetaData getMetaDataFromZk(String metaPath) { + try { + return mapper.readValue(zkApp.getBytes(metaPath), MetaData.class); + } catch (IOException e) { + return null; + } + } + + @Override + public int compareTo(ApplicationFile other) { + if (other == this) return 0; + return this.getPath().getName().compareTo((other).getPath().getName()); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java new file mode 100644 index 00000000000..9061e47d134 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -0,0 +1,281 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.google.common.base.Joiner; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.util.ConfigUtils; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.*; + +/** + * Represents an application residing in zookeeper. + * + * @author tonytv + */ +public class ZKApplicationPackage implements ApplicationPackage { + + private ZKLiveApp liveApp; + + private final Map<Version, PreGeneratedFileRegistry> fileRegistryMap = new HashMap<>(); + private final Map<Version, ProvisionInfo> provisionInfoMap = new HashMap<>(); + private static final Version legacyVersion = Version.fromIntValues(0, 0, 0); + + public static final String fileRegistryNode = "fileregistry"; + public static final String allocatedHostsNode = "allocatedHosts"; + private final ApplicationMetaData metaData; + + public ZKApplicationPackage(ConfigCurator zk, Path appPath) { + verifyAppPath(zk, appPath); + liveApp = new ZKLiveApp(zk, appPath); + metaData = readMetaDataFromLiveApp(liveApp); + importFileRegistries(fileRegistryNode); + importProvisionInfos(allocatedHostsNode); + } + + private void importProvisionInfos(String allocatedHostsNode) { + List<String> provisionInfoNodes = liveApp.getChildren(allocatedHostsNode); + if (provisionInfoNodes.isEmpty()) { + Optional<ProvisionInfo> provisionInfo = importProvisionInfo(allocatedHostsNode); + provisionInfo.ifPresent(info -> provisionInfoMap.put(legacyVersion, info)); + } else { + provisionInfoNodes.stream() + .forEach(versionStr -> { + Version version = Version.fromString(versionStr); + Optional<ProvisionInfo> provisionInfo = importProvisionInfo(Joiner.on("/").join(allocatedHostsNode, versionStr)); + provisionInfo.ifPresent(info -> provisionInfoMap.put(version, info)); + }); + } + } + + private Optional<ProvisionInfo> importProvisionInfo(String provisionInfoNode) { + try { + if (liveApp.exists(provisionInfoNode)) { + return Optional.of(ProvisionInfo.fromJson(liveApp.getBytes(provisionInfoNode))); + } else { + return Optional.empty(); + } + } catch (Exception e) { + throw new RuntimeException("Unable to read provision info", e); + } + } + + private void importFileRegistries(String fileRegistryNode) { + List<String> fileRegistryNodes = liveApp.getChildren(fileRegistryNode); + if (fileRegistryNodes.isEmpty()) { + fileRegistryMap.put(legacyVersion, importFileRegistry(fileRegistryNode)); + } else { + fileRegistryNodes.stream() + .forEach(versionStr -> { + Version version = Version.fromString(versionStr); + fileRegistryMap.put(version, importFileRegistry(Joiner.on("/").join(fileRegistryNode, versionStr))); + }); + } + } + + private PreGeneratedFileRegistry importFileRegistry(String fileRegistryNode) { + try { + return PreGeneratedFileRegistry.importRegistry(liveApp.getDataReader(fileRegistryNode)); + } catch (Exception e) { + throw new RuntimeException("Could not determine which files to distribute. " + + "Please try redeploying the application", e); + } + } + + private ApplicationMetaData readMetaDataFromLiveApp(ZKLiveApp liveApp) { + String metaDataString = liveApp.getData(ConfigCurator.META_ZK_PATH); + if (metaDataString == null || metaDataString.isEmpty()) { + return null; + } + return ApplicationMetaData.fromJsonString(liveApp.getData(ConfigCurator.META_ZK_PATH)); + } + + @Override + public ApplicationMetaData getMetaData() { + return metaData; + } + + private static void verifyAppPath(ConfigCurator zk, Path appPath) { + if (!zk.exists(appPath.getAbsolute())) + throw new RuntimeException("App with path " + appPath + " does not exist"); + } + + @Override + public String getApplicationName() { + return metaData.getApplicationName(); + } + + @Override + public Reader getServices() { + return getUserAppData(SERVICES); + } + + @Override + public Reader getHosts() { + if (liveApp.exists(ConfigCurator.USERAPP_ZK_SUBPATH,HOSTS)) + return getUserAppData(HOSTS); + return null; + } + + @Override + public List<NamedReader> searchDefinitionContents() { + List<NamedReader> ret = new ArrayList<>(); + for (String sd : liveApp.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) { + ret.add(new NamedReader(sd, new StringReader(liveApp.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd)))); + } + } + return ret; + } + + public Map<Version, ProvisionInfo> getProvisionInfoMap() { + return Collections.unmodifiableMap(provisionInfoMap); + } + + @Override + public Map<Version, FileRegistry> getFileRegistryMap() { + return Collections.unmodifiableMap(fileRegistryMap); + } + + private Optional<PreGeneratedFileRegistry> getPreGeneratedFileRegistry(Version vespaVersion) { + // Assumes at least one file registry, which we always have. + Optional<PreGeneratedFileRegistry> fileRegistry = Optional.ofNullable(fileRegistryMap.get(vespaVersion)); + if (!fileRegistry.isPresent()) { + fileRegistry = Optional.of(fileRegistryMap.values().iterator().next()); + } + return fileRegistry; + } + + @Override + public List<NamedReader> getSearchDefinitions() { + return searchDefinitionContents(); + } + + private Reader retrieveConfigDefReader(String def) { + try { + return liveApp.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e); + } + } + + @Override + public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> ret = new LinkedHashMap<>(); + + List<String> allDefs = liveApp.getChildren(ConfigCurator.DEFCONFIGS_ZK_SUBPATH); + + for (final String nodeName : allDefs) { + final ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromZKString(nodeName); + ret.put(key, new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + DefParser parser = new DefParser(key.getName(), retrieveConfigDefReader(nodeName)); + return ConfigDefinitionBuilder.createConfigDefinition(parser.getTree()); + } + + @Override + public String getUnparsedContent() { + try { + return IOUtils.readAll(retrieveConfigDefReader(nodeName)); + } catch (Exception e) { + throw new RuntimeException("Error retriving def file", e); + } + } + }); + } + return ret; + } + + //Returns readers for all the children of a node. + //The node is looked up relative to the location of the active application package + //in zookeeper. + @Override + public List<NamedReader> getFiles(Path relativePath,String suffix,boolean recurse) { + return liveApp.getAllDataFromDirectory(ConfigCurator.USERAPP_ZK_SUBPATH + '/' + relativePath.getRelative(), suffix, recurse); + } + + @Override + public ApplicationFile getFile(Path file) { // foo/bar/baz.json + return new ZKApplicationFile(file, liveApp); + } + + @Override + public String getHostSource() { + return "zookeeper hosts file"; + } + + @Override + public String getServicesSource() { + return "zookeeper services file"; + } + + @Override + public Optional<Reader> getDeployment() { return optionalFile(DEPLOYMENT_FILE.getName()); } + + @Override + public Optional<Reader> getValidationOverrides() { return optionalFile(VALIDATION_OVERRIDES.getName()); } + + private Optional<Reader> optionalFile(String file) { + if (liveApp.exists(ConfigCurator.USERAPP_ZK_SUBPATH, file)) + return Optional.of(getUserAppData(file)); + else + return Optional.empty(); + } + + @Override + public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + List<ComponentInfo> components = new ArrayList<>(); + PreGeneratedFileRegistry fileRegistry = getPreGeneratedFileRegistry(vespaVersion).get(); + for (String path : fileRegistry.getPaths()) { + if (path.startsWith(FilesApplicationPackage.COMPONENT_DIR + File.separator) && path.endsWith(".jar")) { + ComponentInfo component = new ComponentInfo(path); + components.add(component); + } + } + return components; + } + + private Reader getUserAppData(String node) { + return liveApp.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH, node); + } + + @Override + public Reader getRankingExpression(String name) { + return liveApp.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name); + } + + @Override + public File getFileReference(Path pathRelativeToAppDir) { + String fileName = liveApp.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + pathRelativeToAppDir.getRelative()); + return new File(fileName); + } + + @Override + public void validateIncludeDir(String dirName) { + String fullPath = ConfigCurator.USERAPP_ZK_SUBPATH + "/" + dirName; + if (!liveApp.exists(fullPath)) { + throw new IllegalArgumentException("Cannot include directory '" + dirName + + "', as it does not exist in ZooKeeper!"); + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java new file mode 100644 index 00000000000..254203118f2 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java @@ -0,0 +1,208 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; + +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Responsible for providing data from the currently live application subtree in zookeeper. + * (i.e. /vespa/config/apps/<id of currently active app>/) + * + * @author tonytv + */ +public class ZKLiveApp { + + private static final Logger log = Logger.getLogger(ZKLiveApp.class.getName()); + + private final ConfigCurator zk; + private final Path appPath; + + public ZKLiveApp(ConfigCurator zk, Path appPath) { + this.zk = zk; + this.appPath = appPath; + } + + /** + * Returns a list of the files (as readers) in the given path. The readers <b>must</b> + * be closed by the caller. + * + * @param path a path relative to the currently active application (i.e. /vespa/config/apps/<id of currently active app>/). + * @param fileNameSuffix the suffix of files to return, or null to return all + * @param recursive if true, all files from all subdirectories of this will also be returned + * @return the files in the given path, or an empty list (never null) if the directory does not exist or is empty. + * The list gets owned by the caller and can be modified freely. + */ + public List<NamedReader> getAllDataFromDirectory(String path, String fileNameSuffix, boolean recursive) { + return getAllDataFromDirectory(path, "", fileNameSuffix, recursive); + } + + /** + * As above, except + * + * @param namePrefix the prefix to prepend to the returned reader names + */ + private List<NamedReader> getAllDataFromDirectory(String path, String namePrefix, String fileNameSuffix, boolean recursive) { + String fullPath = getFullPath(path); + List<NamedReader> result = new ArrayList<>(); + List<String> children = getChildren(path); + + try { + for (String child : children) { + if (fileNameSuffix == null || child.endsWith(fileNameSuffix)) { + result.add(new NamedReader(namePrefix + child, reader(zk.getData(fullPath, child)))); + if (log.isLoggable(Level.FINER)) + log.finer("ZKApplicationPackage: Added '" + child + "' (matched suffix " + fileNameSuffix + ")"); + } else { + if (log.isLoggable(Level.FINER)) + log.finer("ZKApplicationPackage: Skipped '" + child + "' (did not match suffix " + fileNameSuffix + ")"); + } + if (recursive) + result.addAll(getAllDataFromDirectory(path + "/" + child, namePrefix + child + "/", fileNameSuffix, recursive)); + } + if (log.isLoggable(Level.FINE)) + log.fine("ZKApplicationPackage: Found '" + result.size() + "' files in " + fullPath); + return result; + } catch (Exception e) { + throw new RuntimeException("Could not retrieve all data from '" + fullPath + "' in zookeeper", e); + } + } + + /** + * Retrieves a node relative to the node of the live application, e.g. /vespa/config/apps/$lt;app_id>/<path>/<node> + * + * @param path a path relative to the currently active application + * @param node a path relative to the path above + * @return a Reader that can be used to get the data + */ + public Reader getDataReader(String path, String node) { + final String data = getData(path, node); + if (data == null) { + throw new IllegalArgumentException("No node for " + getFullPath(path) + "/" + node + " exists"); + } + return reader(data); + } + + public String getData(String path, String node) { + try { + return zk.getData(getFullPath(path), node); + } catch (Exception e) { + throw new IllegalArgumentException("Could not retrieve node '" + getFullPath(path) + "/" + node + "' in zookeeper", e); + } + } + + public String getData(String path) { + try { + return zk.getData(getFullPath(path)); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Could not retrieve path '" + getFullPath(path) + "' in zookeeper", e); + } + } + + public byte[] getBytes(String path) { + try { + return zk.getBytes(getFullPath(path)); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Could not retrieve path '" + getFullPath(path) + "' in zookeeper", e); + } + } + + public void putData(String path, String data) { + try { + zk.putData(getFullPath(path), data); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Could not put data to node '" + getFullPath(path) + "' in zookeeper", e); + } + } + + public void create(String path, String node) { + if (path != null && !path.startsWith("/")) path = "/" + path; + try { + zk.createNode(getFullPath(path), node); + } catch (RuntimeException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Checks if the given node exists under path under this live app + * + * @param path a zookeeper path + * @param node a zookeeper node + * @return true if the node exists in the path, false otherwise + */ + public boolean exists(String path, String node) { + return zk.exists(getFullPath(path), node); + } + + /** + * Checks if the given node exists under path under this live app + * + * @param path a zookeeper path + * @return true if the node exists in the path, false otherwise + */ + public boolean exists(String path) { + return zk.exists(getFullPath(path)); + } + + private String getFullPath(String path) { + Path fullPath = appPath; + if (path != null) { + fullPath = appPath.append(path); + } + return fullPath.getAbsolute(); + } + + /** + * Recursively delete given path + * + * @param path path to delete + */ + public void deleteRecurse(String path) { + zk.deleteRecurse(getFullPath(path)); + } + + /** + * Returns the full list of children (file names) in the given path. + * + * @param path a path relative to the currently active application + * @return a list of file names + */ + public List<String> getChildren(String path) { + String fullPath = getFullPath(path); + if (! zk.exists(fullPath)) { + log.fine("ZKApplicationPackage: " + fullPath + " is not a valid dir"); + return Collections.emptyList(); + } + return zk.getChildren(fullPath); + } + + private static Reader reader(String string) { + return new StringReader(string); + } + + public void create(String path) { + if (path != null && !path.startsWith("/")) path = "/" + path; + try { + zk.createNode(getFullPath(path)); + } catch (RuntimeException e) { + throw new IllegalArgumentException(e); + } + } + + public Reader getDataReader(String path) { + final String data = getData(path); + if (data == null) { + throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists"); + } + return reader(data); + } +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/package-info.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/package-info.java new file mode 100644 index 00000000000..a548df4d493 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author tonytv + */ +@ExportPackage +package com.yahoo.vespa.config.server.zookeeper; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/Cluster.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/Cluster.java new file mode 100644 index 00000000000..879dae419d2 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/Cluster.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import java.util.Arrays; +import java.util.List; + +import com.google.common.collect.ImmutableList; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Model a single cluster of services in the Vespa model. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Cluster implements Comparable<Cluster> { + @NonNull + public final String name; + @NonNull + public final String type; + /** + * An ordered list of the service instances in this cluster. + */ + @NonNull + public final ImmutableList<Service> services; + + public Cluster(String name, String type, List<Service> services) { + this.name = name; + this.type = type; + ImmutableList.Builder<Service> builder = ImmutableList.builder(); + Service[] sortingBuffer = services.toArray(new Service[0]); + Arrays.sort(sortingBuffer); + builder.add(sortingBuffer); + this.services = builder.build(); + } + + @Override + public int compareTo(Cluster other) { + int nameOrder = name.compareTo(other.name); + if (nameOrder != 0) { + return nameOrder; + } + return type.compareTo(other.type); + } + + @Override + public int hashCode() { + final int prime = 761; + int result = 1; + result = prime * result + name.hashCode(); + result = prime * result + type.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Cluster other = (Cluster) obj; + if (!name.equals(other.name)) { + return false; + } + if (!type.equals(other.type)) { + return false; + } + return true; + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("Cluster [name=").append(name).append(", type=").append(type).append(", services=") + .append(services.subList(0, Math.min(services.size(), maxLen))).append("]"); + return builder.toString(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java new file mode 100644 index 00000000000..163743cb1bb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/ConfigServerLocation.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import com.yahoo.cloud.config.ConfigserverConfig; + +/** + * Wrapper for settings from the cloud.config.configserver config. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ConfigServerLocation { + public final int restApiPort; + + public ConfigServerLocation(ConfigserverConfig configServer) { + restApiPort = configServer.httpport(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ConfigServerLocation [restApiPort=").append(restApiPort).append("]"); + return builder.toString(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ProxyErrorMapper.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ProxyErrorMapper.java new file mode 100644 index 00000000000..60055df88df --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/ProxyErrorMapper.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +/** + * Convert exceptions thrown by the internal REST client into a little more helpful responses. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@Provider +public class ProxyErrorMapper implements ExceptionMapper<WebApplicationException> { + + @Override + public Response toResponse(WebApplicationException exception) { + StringBuilder msg = new StringBuilder("Invoking (external) web service failed: "); + msg.append(exception.getMessage()); + return Response.status(500).entity(msg.toString()).type("text/plain").build(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/Service.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/Service.java new file mode 100644 index 00000000000..4d0dc36a0c7 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/Service.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.yahoo.text.Utf8; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Model a single service instance as a sortable object. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Service implements Comparable<Service> { + @NonNull + public final String serviceType; + @NonNull + public final String host; + public final int statePort; + @NonNull + public final String configId; + @NonNull + public final List<Integer> ports; + @NonNull + public final String name; + + public Service(String serviceType, String host, int statePort, String clusterName, String clusterType, + String configId, List<Integer> ports, String name) { + this.serviceType = serviceType; + this.host = host.toLowerCase(); + this.statePort = statePort; + this.configId = configId; + ImmutableList.Builder<Integer> portsBuilder = new ImmutableList.Builder<>(); + portsBuilder.addAll(ports); + this.ports = portsBuilder.build(); + this.name = name; + } + + @Override + public int compareTo(Service other) { + int serviceTypeOrder = serviceType.compareTo(other.serviceType); + if (serviceTypeOrder != 0) { + return serviceTypeOrder; + } + int hostOrder = host.compareTo(other.host); + if (hostOrder != 0) { + return hostOrder; + } + return Integer.compare(statePort, other.statePort); + } + + /** + * Generate an identifier string for one of the ports of this service + * suitable for using in an URL. + * + * @param port + * port which this identifier pertains to + * @return an opaque identifier string for this service + */ + public String getIdentifier(int port) { + StringBuilder b = new StringBuilder(serviceType); + b.append("-"); + MessageDigest md5; + try { + md5 = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("MD5 should by definition always be available in the JVM.", e); + } + md5.update(Utf8.toBytes(serviceType)); + md5.update(Utf8.toBytes(configId)); + md5.update(Utf8.toBytes(host)); + for (int i = 3; i >= 0; --i) { + md5.update((byte) (port >>> i)); + } + byte[] digest = md5.digest(); + BigInteger digestMarshal = new BigInteger(1, digest); + b.append(digestMarshal.toString(36)); + return b.toString(); + } + + /** + * All valid identifiers for this object. + * + * @return a list with a unique ID for each of this service's ports + */ + public List<String> getIdentifiers() { + List<String> ids = new ArrayList<>(ports.size()); + for (int port : ports) { + ids.add(getIdentifier(port)); + } + return ids; + } + + /** + * Find which port number a hash code pertains to. + * + * @param identifier a string generated from {@link #getIdentifier(int)} + * @return a port number, or 0 if no match is found + */ + public int matchIdentifierWithPort(String identifier) { + for (int port : ports) { + if (identifier.equals(getIdentifier(port))) { + return port; + } + } + throw new IllegalArgumentException("Identifier " + identifier + " matches no ports in " + this); + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("Service [serviceType=").append(serviceType).append(", host=").append(host).append(", statePort=") + .append(statePort).append(", configId=").append(configId).append(", ports=") + .append(ports.subList(0, Math.min(ports.size(), maxLen))).append(", name=").append(name) + .append("]"); + return builder.toString(); + } + + @Override + public int hashCode() { + final int prime = 131; + int result = 1; + result = prime * result + configId.hashCode(); + result = prime * result + host.hashCode(); + result = prime * result + name.hashCode(); + result = prime * result + ports.hashCode(); + result = prime * result + serviceType.hashCode(); + result = prime * result + statePort; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Service other = (Service) obj; + if (!configId.equals(other.configId)) { + return false; + } + if (!host.equals(other.host)) { + return false; + } + if (!name.equals(other.name)) { + return false; + } + if (!ports.equals(other.ports)) { + return false; + } + if (!serviceType.equals(other.serviceType)) { + return false; + } + if (statePort != other.statePort) { + return false; + } + return true; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java new file mode 100644 index 00000000000..12141366811 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/ServiceModel.java @@ -0,0 +1,236 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Table; +import com.google.common.collect.Table.Cell; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.ClusterView; +import com.yahoo.vespa.serviceview.bindings.HostService; +import com.yahoo.vespa.serviceview.bindings.ModelResponse; +import com.yahoo.vespa.serviceview.bindings.ServicePort; +import com.yahoo.vespa.serviceview.bindings.ServiceView; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * A transposed view for cloud.config.model. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ServiceModel { + private static final String CLUSTERCONTROLLER_TYPENAME = "container-clustercontroller"; + + private static final String CONTENT_CLUSTER_TYPENAME = "content"; + + private final Map<String, Service> servicesMap; + + /** + * An ordered list of the clusters in this config model. + */ + @NonNull + public final ImmutableList<Cluster> clusters; + + ServiceModel(ModelResponse modelConfig) { + Table<String, String, List<Service>> services = HashBasedTable.create(); + for (HostService h : modelConfig.hosts) { + String hostName = h.name; + for (com.yahoo.vespa.serviceview.bindings.Service s : h.services) { + addService(services, hostName, s); + } + } + List<Cluster> sortingBuffer = new ArrayList<>(); + for (Cell<String, String, List<Service>> c : services.cellSet()) { + sortingBuffer.add(new Cluster(c.getRowKey(), c.getColumnKey(), c.getValue())); + } + Collections.sort(sortingBuffer); + ImmutableList.Builder<Cluster> clustersBuilder = new ImmutableList.Builder<>(); + clustersBuilder.addAll(sortingBuffer); + clusters = clustersBuilder.build(); + Map<String, Service> seenIdentifiers = new HashMap<>(); + for (Cluster c : clusters) { + for (Service s : c.services) { + List<String> identifiers = s.getIdentifiers(); + for (String identifier : identifiers) { + if (seenIdentifiers.containsKey(identifier)) { + throw new RuntimeException( + "Congrats, you have a publishable result. We have a very unexpected hash collision" + " between " + + seenIdentifiers.get(identifier) + " and " + s + "."); + } + seenIdentifiers.put(identifier, s); + } + } + } + ImmutableMap.Builder<String, Service> servicesBuilder = new ImmutableMap.Builder<>(); + servicesBuilder.putAll(seenIdentifiers); + servicesMap = servicesBuilder.build(); + } + + private static void addService(Table<String, String, List<Service>> services, + String hostName, + com.yahoo.vespa.serviceview.bindings.Service s) { + boolean hasStateApi = false; + int statePort = 0; + List<Integer> ports = new ArrayList<>(s.ports.size()); + for (ServicePort port : s.ports) { + ports.add(port.number); + if (!hasStateApi && port.hasTags("http", "state")) { + hasStateApi = true; + statePort = port.number; + } + } + // ignore hosts without state API + if (hasStateApi) { + Service service = new Service(s.type, hostName, statePort, s.clustername, s.clustertype, s.configid, ports, s.name); + getAndSetEntry(services, s.clustername, s.clustertype).add(service); + } + } + + private static List<Service> getAndSetEntry(Table<String, String, List<Service>> services, String clusterName, String clusterType) { + List<Service> serviceList = services.get(clusterName, clusterType); + if (serviceList == null) { + serviceList = new ArrayList<>(); + services.put(clusterName, clusterType, serviceList); + } + return serviceList; + } + + /** + * The top level view of a given application. + * + * @return a top level view of the entire application in a form suitable for + * consumption by a REST API + */ + public ApplicationView showAllClusters(String uriBase, String applicationIdentifier) { + ApplicationView response = new ApplicationView(); + List<ClusterView> clusterViews = new ArrayList<>(); + for (Cluster c : clusters) { + clusterViews.add(showCluster(c, uriBase, applicationIdentifier)); + } + response.clusters = clusterViews; + return response; + } + + private ClusterView showCluster(Cluster c, String uriBase, String applicationIdentifier) { + List<ServiceView> services = new ArrayList<>(); + for (Service s : c.services) { + ServiceView service = new ServiceView(); + StringBuilder buffer = getLinkBuilder(uriBase).append(applicationIdentifier).append('/'); + service.url = buffer.append("service/").append(s.getIdentifier(s.statePort)).append("/state/v1/").toString(); + service.serviceType = s.serviceType; + service.serviceName = s.name; + service.configId = s.configId; + service.host = s.host; + addLegacyLink(uriBase, applicationIdentifier, s, service); + services.add(service); + } + ClusterView v = new ClusterView(); + v.services = services; + v.name = c.name; + v.type = c.type; + if (CONTENT_CLUSTER_TYPENAME.equals(c.type)) { + Service s = getFirstClusterController(); + StringBuilder buffer = getLinkBuilder(uriBase).append(applicationIdentifier).append('/'); + buffer.append("service/").append(s.getIdentifier(s.statePort)).append("/cluster/v2/").append(c.name); + v.url = buffer.toString(); + } else { + v.url = null; + } + return v; + } + + private void addLegacyLink(String uriBase, String applicationIdentifier, Service s, ServiceView service) { + if (s.serviceType.equals("storagenode") || s.serviceType.equals("distributor")) { + StringBuilder legacyBuffer = getLinkBuilder(uriBase); + legacyBuffer.append("legacy/").append(applicationIdentifier).append('/'); + legacyBuffer.append("service/").append(s.getIdentifier(s.statePort)).append('/'); + service.legacyStatusPages = legacyBuffer.toString(); + } + } + + private Service getFirstServiceInstanceByType(@NonNull String typeName) { + for (Cluster c : clusters) { + for (Service s : c.services) { + if (typeName.equals(s.serviceType)) { + return s; + } + } + } + throw new IllegalStateException("This installation has but no service of required type: " + + typeName + "."); + } + + private Service getFirstClusterController() { + // This is used assuming all cluster controllers know of all fleet controllers in an application + return getFirstServiceInstanceByType(CLUSTERCONTROLLER_TYPENAME); + } + + private StringBuilder getLinkBuilder(String uriBase) { + StringBuilder buffer = new StringBuilder(uriBase); + if (!uriBase.endsWith("/")) { + buffer.append('/'); + } + return buffer; + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("ServiceModel [clusters=") + .append(clusters.subList(0, Math.min(clusters.size(), maxLen))).append("]"); + return builder.toString(); + } + + + /** + * Match an identifier with a service for this cluster. + * + * @param identifier + * an opaque service identifier generated by the service + * @return the corresponding Service instance + */ + public Service getService(String identifier) { + return servicesMap.get(identifier); + } + + /** + * Find a service based on host and port. + * + * @param host + * the name of the host running the service + * @param port + * a port owned by the service + * @param self + * the service which generated the host data + * @return a service instance fullfilling the criteria + * @throws IllegalArgumentException + * if no matching service is found + */ + public Service resolve(String host, int port, Service self) { + Integer portAsObject = Integer.valueOf(port); + String realHost; + if ("localhost".equals(host)) { + realHost = self.host; + } else { + realHost = host; + } + for (Cluster c : clusters) { + for (Service s : c.services) { + if (s.host.equals(realHost) && s.ports.contains(portAsObject)) { + return s; + } + } + } + throw new IllegalArgumentException("No registered service owns port " + port + " on host " + realHost + "."); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java new file mode 100644 index 00000000000..c66e0c2b3ba --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateResource.java @@ -0,0 +1,282 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.UriInfo; + +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.ConfigClient; +import com.yahoo.vespa.serviceview.bindings.HealthClient; +import com.yahoo.vespa.serviceview.bindings.ModelResponse; +import com.yahoo.vespa.serviceview.bindings.StateClient; + +import org.glassfish.jersey.client.proxy.WebResourceFactory; + + +/** + * A web service to discover and proxy Vespa service state info. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@Path("/") +public class StateResource implements StateClient { + private static final String SINGLE_API_LINK = "url"; + private final int restApiPort; + private final String host; + private final UriInfo uriInfo; + + @SuppressWarnings("serial") + private static class GiveUpLinkRetargetingException extends Exception { + public GiveUpLinkRetargetingException(Throwable reason) { + super(reason); + } + + public GiveUpLinkRetargetingException(String message) { + super(message); + } + } + + public StateResource(@Component ConfigServerLocation configServer, @Context UriInfo ui) { + this.restApiPort = configServer.restApiPort; + host = "localhost"; + this.uriInfo = ui; + } + + @Override + @GET + @Path("v1/") + @Produces(MediaType.APPLICATION_JSON) + public ApplicationView getDefaultUserInfo() { + return getUserInfo("default", "default", "default", "default", "default"); + } + + @Override + @GET + @Path("v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}") + @Produces(MediaType.APPLICATION_JSON) + public ApplicationView getUserInfo(@PathParam("tenantName") String tenantName, + @PathParam("applicationName") String applicationName, + @PathParam("environmentName") String environmentName, + @PathParam("regionName") String regionName, + @PathParam("instanceName") String instanceName) { + ServiceModel model = new ServiceModel( + getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); + return model.showAllClusters( + getBaseUri() + "v1/", + applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName)); + } + + + @Produces(MediaType.TEXT_HTML) + public interface HtmlProxyHack { + @GET + public String proxy(); + } + + @GET + @Path("v1/legacy/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}/service/{serviceIdentifier}/{apiParams: .*}") + @Produces(MediaType.TEXT_HTML) + public String htmlProxy(@PathParam("tenantName") String tenantName, + @PathParam("applicationName") String applicationName, + @PathParam("environmentName") String environmentName, + @PathParam("regionName") String regionName, + @PathParam("instanceName") String instanceName, + @PathParam("serviceIdentifier") String identifier, + @PathParam("apiParams") String apiParams) { + ServiceModel model = new ServiceModel(getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); + Service s = model.getService(identifier); + int requestedPort = s.matchIdentifierWithPort(identifier); + Client client = ClientBuilder.newClient(); + try { + final StringBuilder uriBuffer = new StringBuilder("http://").append(s.host).append(':').append(requestedPort).append('/') + .append(apiParams); + addQuery(uriBuffer); + String uri = uriBuffer.toString(); + WebTarget target = client.target(uri); + HtmlProxyHack resource = WebResourceFactory.newResource(HtmlProxyHack.class, target); + return resource.proxy(); + } finally { + if (client != null) { + client.close(); + } + } + } + + private String getBaseUri() { + String baseUri = uriInfo.getBaseUri().toString(); + if (baseUri.endsWith("/")) { + return baseUri; + } else { + return baseUri + "/"; + } + } + + protected ModelResponse getModelConfig(String tenant, String application, String environment, String region, String instance) { + Client client = ClientBuilder.newClient(); + try { + WebTarget target = client.target("http://" + host + ":" + restApiPort + "/"); + + ConfigClient resource = WebResourceFactory.newResource(ConfigClient.class, target); + + return resource.getServiceModel(tenant, application, environment, region, instance); + } finally { + if (client != null) { + client.close(); + } + } + } + + @SuppressWarnings("rawtypes") + @Override + @GET + @Path("v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}/service/{serviceIdentifier}/{apiParams: .*}") + @Produces(MediaType.APPLICATION_JSON) + public HashMap singleService(@PathParam("tenantName") String tenantName, + @PathParam("applicationName") String applicationName, + @PathParam("environmentName") String environmentName, + @PathParam("regionName") String regionName, + @PathParam("instanceName") String instanceName, + @PathParam("serviceIdentifier") String identifier, + @PathParam("apiParams") String apiParams) { + ServiceModel model = new ServiceModel(getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); + Service s = model.getService(identifier); + int requestedPort = s.matchIdentifierWithPort(identifier); + Client client = ClientBuilder.newClient(); + try { + HealthClient resource = getHealthClient(apiParams, s, requestedPort, client); + HashMap<?, ?> apiResult = resource.getHealthInfo(); + rewriteResourceLinks(apiResult, model, s, applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName), identifier); + return apiResult; + } finally { + if (client != null) { + client.close(); + } + } + } + + protected HealthClient getHealthClient(String apiParams, Service s, int requestedPort, Client client) { + final StringBuilder uriBuffer = new StringBuilder("http://").append(s.host).append(':').append(requestedPort).append('/') + .append(apiParams); + addQuery(uriBuffer); + WebTarget target = client.target(uriBuffer.toString()); + return WebResourceFactory.newResource(HealthClient.class, target); + } + + private String applicationIdentifier(String tenant, String application, String environment, String region, String instance) { + return new StringBuilder("tenant/").append(tenant).append("/application/").append(application).append("/environment/") + .append(environment).append("/region/").append(region).append("/instance/").append(instance).toString(); + } + + private void rewriteResourceLinks(Object apiResult, + ServiceModel model, + Service self, + String applicationIdentifier, + String incomingIdentifier) { + if (apiResult instanceof List) { + for (@SuppressWarnings("unchecked") ListIterator<Object> i = ((List<Object>) apiResult).listIterator(); i.hasNext();) { + Object resource = i.next(); + if (resource instanceof String) { + try { + StringBuilder buffer = linkBuffer(applicationIdentifier); + // if it points to a port and host not part of the application, rewriting will not occur, so this is kind of safe + retarget(model, self, buffer, (String) resource); + i.set(buffer.toString()); + } catch (GiveUpLinkRetargetingException e) { + break; // assume relatively homogenous lists when doing rewrites to avoid freezing up on scanning long lists + } + } else { + rewriteResourceLinks(resource, model, self, applicationIdentifier, incomingIdentifier); + } + } + } else if (apiResult instanceof Map) { + @SuppressWarnings("unchecked") + Map<Object, Object> api = (Map<Object, Object>) apiResult; + for (Map.Entry<Object, Object> entry : api.entrySet()) { + if (SINGLE_API_LINK.equals(entry.getKey()) && entry.getValue() instanceof String) { + try { + rewriteSingleLink(entry, model, self, linkBuffer(applicationIdentifier)); + } catch (GiveUpLinkRetargetingException e) { + // NOP + } + } else if ("link".equals(entry.getKey()) && entry.getValue() instanceof String) { + buildSingleLink(entry, model, linkBuffer(applicationIdentifier), incomingIdentifier); + } else { + rewriteResourceLinks(entry.getValue(), model, self, applicationIdentifier, incomingIdentifier); + } + } + } + } + + private void buildSingleLink(Map.Entry<Object, Object> entry, + ServiceModel model, + StringBuilder newUri, + String incomingIdentifier) { + newUri.append("/service/") + .append(incomingIdentifier); + newUri.append(entry.getValue()); + entry.setValue(newUri.toString()); + } + + private void addQuery(StringBuilder newUri) { + String query = uriInfo.getRequestUri().getRawQuery(); + if (query != null && query.length() > 0) { + newUri.append('?').append(query); + } + } + + private StringBuilder linkBuffer(String applicationIdentifier) { + StringBuilder newUri = new StringBuilder(getBaseUri()); + newUri.append("v1/").append(applicationIdentifier); + return newUri; + } + + private void rewriteSingleLink(Map.Entry<Object, Object> entry, + ServiceModel model, + Service self, + StringBuilder newUri) throws GiveUpLinkRetargetingException { + String url = (String) entry.getValue(); + retarget(model, self, newUri, url); + entry.setValue(newUri.toString()); + } + + private void retarget(ServiceModel model, Service self, StringBuilder newUri, String url) throws GiveUpLinkRetargetingException { + URI link; + try { + link = new URI(url); + } catch (URISyntaxException e) { + throw new GiveUpLinkRetargetingException(e); + } + if (!link.isAbsolute()) { + throw new GiveUpLinkRetargetingException("This rewriting only supports absolute URIs."); + } + int linkPort = link.getPort(); + if (linkPort == -1) { + linkPort = 80; + } + Service s; + try { + s = model.resolve(link.getHost(), linkPort, self); + } catch (IllegalArgumentException e) { + throw new GiveUpLinkRetargetingException(e); + } + newUri.append("/service/").append(s.getIdentifier(linkPort)); + newUri.append(link.getPath()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/package-info.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/package-info.java new file mode 100644 index 00000000000..5cc9fa85775 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/package-info.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Home of the centralised service view implementation. The service view is a + * REST API for discovering and accessing the state API for any service in a + * Vespa cluster. + * + * <p>Do note this package is in its prototyping stage and classes <i>will</i> + * be renamed and moved around a little.</p> + */ +@ExportPackage +package com.yahoo.vespa.serviceview; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml new file mode 100644 index 00000000000..1b86aed3983 --- /dev/null +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -0,0 +1,129 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0" xmlns:preprocess="properties"> + <jdisc id="configserver" jetty="true" version="1.0"> + <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <component id="com.yahoo.vespa.config.server.ConfigServerBootstrap" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.monitoring.Metrics" bundle="configserver" /> + <component id="com.yahoo.vespa.zookeeper.ZooKeeperServer" bundle="zkfacade" /> + <component id="com.yahoo.vespa.config.server.RpcServer" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.ConfigServerDB" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.session.FileDistributionFactory" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.SuperModelGenerationCounter" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.session.SessionPreparer" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.SuperModelController" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.StaticConfigDefinitionRepo" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.provision.HostProvisionerProvider" bundle="configserver" /> + <component id="com.yahoo.vespa.curator.Curator" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.InjectedGlobalComponentRegistry" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.Tenants" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.application.PermanentApplicationPackage" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.HostRegistries" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.deploy.Deployer" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.zookeeper.ConfigCurator" bundle="configserver" /> + <component id="com.yahoo.container.jdisc.metric.state.StateMetricConsumerFactory" bundle="container-disc" /> + <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> + <component id="com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker" bundle="configserver" /> + + <component id="com.yahoo.vespa.serviceview.ConfigServerLocation" bundle="configserver" /> + + <components> + <include dir="config-models" /> + </components> + + <preprocess:include file='config-models.xml' required='false' /> + <preprocess:include file='node-repository.xml' required='false' /> + <preprocess:include file='hosted-vespa/routing-status.xml' required='false' /> + <component id="com.yahoo.vespa.service.monitor.SlobrokAndConfigIntersector" bundle="orchestrator" /> + <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" /> + <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" /> + <component id="com.yahoo.vespa.orchestrator.controller.RetryingClusterControllerClientFactory" bundle="orchestrator" /> + <component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" /> + + <rest-api path="orchestrator" jersey2="true"> + <components bundle="orchestrator" /> + </rest-api> + + <rest-api path="serviceview" jersey2="true"> + <components bundle="configserver"> + <package>com.yahoo.vespa.serviceview</package> + </components> + </rest-api> + + <rest-api path="status" jersey2="true"> + <components bundle="configserver"> + <package>com.yahoo.vespa.config.server.restapi.impl</package> + <package>com.yahoo.vespa.config.server.restapi.resources</package> + </components> + </rest-api> + + <handler id='com.yahoo.vespa.config.server.http.HttpGetConfigHandler' bundle='configserver'> + <binding>http://*/config/v1/*/*</binding> + <binding>http://*/config/v1/*</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.HttpListConfigsHandler' bundle='configserver'> + <binding>http://*/config/v1/</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.HttpListNamedConfigsHandler' bundle='configserver'> + <binding>http://*/config/v1/*/</binding> + <binding>http://*/config/v1/*/*/</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.ListTenantsHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.TenantHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.SessionCreateHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*/session</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.SessionPrepareHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*/session/*/prepared</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.SessionActiveHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*/session/*/active</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.SessionContentHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*/session/*/content/*</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.ListApplicationsHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*/application/</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.ApplicationHandler' bundle='configserver'> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/log</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/converge</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*</binding> + <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*</binding> + <binding>http://*/application/v2/tenant/*/application/*</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.HttpGetConfigHandler' bundle='configserver'> + <binding>http://*/config/v2/tenant/*/application/*/*</binding> + <binding>http://*/config/v2/tenant/*/application/*/*/*</binding> + <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*</binding> + <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler' bundle='configserver'> + <binding>http://*/config/v2/tenant/*/application/*/</binding> + <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.HttpListNamedConfigsHandler' bundle='configserver'> + <binding>http://*/config/v2/tenant/*/application/*/*/</binding> + <binding>http://*/config/v2/tenant/*/application/*/*/*/</binding> + <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/</binding> + <binding>http://*/config/v2/tenant/*/application/*/environment/*/region/*/instance/*/*/*/</binding> + </handler> + <handler id='com.yahoo.vespa.config.server.http.v2.HostHandler' bundle='configserver'> + <binding>http://*/application/v2/host/*</binding> + </handler> + + <http> + <server port="19071" id="configserver" /> + <preprocess:include file='hosted-vespa/http-server.xml' required='false' /> + </http> + </jdisc> +</services> diff --git a/configserver/src/main/resources/logd/logd.cfg b/configserver/src/main/resources/logd/logd.cfg new file mode 100644 index 00000000000..a4677deb3f6 --- /dev/null +++ b/configserver/src/main/resources/logd/logd.cfg @@ -0,0 +1,9 @@ +logserver.use false +loglevel.fatal.forward false +loglevel.error.forward false +loglevel.warning.forward false +loglevel.config.forward false +loglevel.info.forward false +loglevel.event.forward false +loglevel.debug.forward false +loglevel.spam.forward false diff --git a/configserver/src/main/sh/cloudconfig_server-remove-state b/configserver/src/main/sh/cloudconfig_server-remove-state new file mode 100755 index 00000000000..3ec76aaea92 --- /dev/null +++ b/configserver/src/main/sh/cloudconfig_server-remove-state @@ -0,0 +1,163 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME +cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } + +usage() { + ( + echo "This script will remove cloudconfig_server state on this node." + echo "It will refuse to execute if cloudconfig_server is running." + echo "The following options are recognized:" + echo "" + + echo "-force do not ask for confirmation before removal" + ) >&2 +} + +sudo="sudo" +ask=true +remove_zookeeper_dir=true +remove_applications_dir=true +remove_tenants_dir=true +confirmed=true +zookeeper_dir=var/zookeeper +applications_dir=var/db/vespa/config_server/serverdb/applications +tenants_dir=var/db/vespa/config_server/serverdb/tenants + +if [ -w $applications_dir ] && [ -w $zookeeper_dir ]; then + sudo="" +fi + +while [ $# -gt 0 ]; do + case $1 in + -h|-help) usage; exit 0;; + -nosudo) shift; sudo="" ;; + -sudo) shift; sudo="sudo" ;; + -force) shift; ask=false ;; + *) echo "Unrecognized option '$1'" >&2; usage; exit 1;; + esac +done +# Will first check if cloudconfig_server is running on this node +P_CONFIGSERVER=var/run/configserver.pid +if [ -f $P_CONFIGSERVER ] && $sudo kill -0 `cat $P_CONFIGSERVER` 2>/dev/null; then + echo "[ERROR] Will not remove indexes while cloudconfig_server is running" 1>&2 + echo "[ERROR] 'stop cloudconfig_server' and 'ps xgauww' to check for cloudconfig_server process" 1>&2 + exit 1 +fi + +removedata() { + echo "[info] removing data: $sudo rm -rf $*" + $sudo rm -rf $* + echo "[info] removed." +} + +confirm() { + confirmed=false + echo -n 'Really remove state for cloudconfig_server in '$ROOT/$1'? Type "yes" if you are sure ==> ' 1>&2 + answer=no + read answer + if [ "$answer" = "yes" ]; then + confirmed=true + else + confirmed=false + echo "[info] skipping removal ('$answer' != 'yes')" + fi +} + +garbage_collect_dirs() { + find $zookeeper_dir $applications_dir -type d -depth 2>/dev/null | while read dir; do + [ "$dir" = "$zookeeper_dir" ] && continue + [ "$dir" = "$applications_dir" ] && continue + $sudo rmdir "$dir" 2>/dev/null + done +} + +confirm_and_clean_dir() { + if $ask; then + kb=$(du -sk $1 | awk '{print $1}') + if [ $kb -gt 100 ]; then + confirm $1 + fi + fi + if $confirmed; then + removedata $1/* + fi +} + +garbage_collect_dirs + +if $remove_zookeeper_dir && [ -d $zookeeper_dir ]; then + confirm_and_clean_dir $zookeeper_dir +fi + +if $remove_applications_dir && [ -d $applications_dir ]; then + confirm_and_clean_dir $applications_dir +fi + +if $remove_tenants_dir && [ -d $tenants_dir ]; then + confirm_and_clean_dir $tenants_dir +fi + +garbage_collect_dirs + +exit 0 diff --git a/configserver/src/main/sh/ping-configserver b/configserver/src/main/sh/ping-configserver new file mode 100755 index 00000000000..b14fc22a9ae --- /dev/null +++ b/configserver/src/main/sh/ping-configserver @@ -0,0 +1,9 @@ +#!/bin/sh +host=$1 +port=$2 +curl -s -S -m 5 http://$host:$port/state/v1/health | grep "status\": {\"code\": \"up\"}" +if [ $? -gt 0 ]; then + exit 1 +else + exit 0 +fi diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver new file mode 100755 index 00000000000..358454bd5bd --- /dev/null +++ b/configserver/src/main/sh/start-configserver @@ -0,0 +1,119 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME +export ROOT +cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } + +# get common PATH etc: +. $ROOT/libexec/vespa/common-env.sh + +if [ -f $ROOT/conf/zookeeper/zookeeper.cfg ]; then + chown yahoo $ROOT/conf/zookeeper/zookeeper.cfg + chmod 644 $ROOT/conf/zookeeper/zookeeper.cfg +fi + +if [ -f $ROOT/var/zookeeper/myid ]; then + chown yahoo $ROOT/var/zookeeper/myid + chmod 644 $ROOT/var/zookeeper/myid +fi + +$ROOT/libexec/vespa/vespa-config.pl -isthisaconfigserver 1>/dev/null +if [ "$?" != "0" ] ; then + echo "Not able to start config server, host `hostname` is not part of 'services.addr_configserver'" + exit 1; +fi + +fixlimits +checkjava + +ZOOKEEPER_DATA_PATH="$VESPA_HOME/var/zookeeper/version-2" +if [ ! -d "$ZOOKEEPER_DATA_PATH" ]; then + echo "Creating data directory $ZOOKEEPER_DATA_PATH" + mkdir -p $ZOOKEEPER_DATA_PATH + chown yahoo:users $ZOOKEEPER_DATA_PATH +fi + +ZOOKEEPER_LOG_FILE="$VESPA_HOME/logs/vespa/zookeeper.configserver.log" +rm -f $ZOOKEEPER_LOG_FILE*lck + +baseuserargs=$vespa_base__jvmargs_configserver +serveruserargs="$cloudconfig_server__jvmargs" + +# TODO: Move this stuff to package when fully working +APP=$VESPA_HOME/conf/configserver-app +VESPA_SERVICE_NAME=configserver +LOGFILE="${ROOT}/logs/vespa/vespa.log" +VESPA_LOG_TARGET="file:${LOGFILE}" +PID_FILE="${ROOT}/var/run/configserver.pid" +VESPA_LOG_CONTROL_DIR="$ROOT/var/db/vespa/logcontrol" +VESPA_LOG_CONTROL_FILE="$ROOT/var/db/vespa/logcontrol/configserver.logcontrol" +jvmargs="-Dzookeeperlogfile=$ZOOKEEPER_LOG_FILE $baseuserargs $serveruserargs" +standalone_jdisc_container__deployment_profile="configserver" +MAXRESTARTS=100000 +export UNPRIVILEGED=1 +export MALLOC_ARENA_MAX=1 #Does not need fast allocation +export VESPA_SERVICE_NAME VESPA_LOG_TARGET PID_FILE VESPA_LOG_CONTROL_DIR JAVA_OPTS VESPA_LOG_CONTROL_FILE standalone_jdisc_container__deployment_profile MAXRESTARTS +run-as-yahoo ${ROOT}/bin/jdisc_container_start $APP $jvmargs + +run-as-yahoo $ROOT/libexec/vespa/start-filedistribution +run-as-yahoo $ROOT/libexec/vespa/start-logd diff --git a/configserver/src/main/sh/start-filedistribution b/configserver/src/main/sh/start-filedistribution new file mode 100755 index 00000000000..a62afce40b7 --- /dev/null +++ b/configserver/src/main/sh/start-filedistribution @@ -0,0 +1,82 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME + +VESPA_CONFIG_ID="dir:${ROOT}/conf/filedistributor" +export VESPA_CONFIG_ID + +if [ "$multitenant" = "true" ]; then + foo=`${ROOT}/libexec/vespa/vespa-config.pl -mkfiledistributorconfig` + PIDFILE_FILEDISTRIBUTOR=var/run/filedistributor.pid + LOGFILE="${ROOT}/logs/vespa/vespa.log" + VESPA_LOG_TARGET="file:${LOGFILE}" + VESPA_LOG_CONTROL_DIR="${ROOT}/var/db/vespa/logcontrol" + VESPA_LOG_CONTROL_FILE="${ROOT}/var/db/vespa/logcontrol/filedistributor.logcontrol" + VESPA_SERVICE_NAME=filedistributor + export VESPA_SERVICE_NAME + export VESPA_LOG_TARGET + export VESPA_LOG_CONTROL_DIR + export VESPA_LOG_CONTROL_FILE + cd ${ROOT} + vespa-runserver -r 30 -s filedistributor -p $PIDFILE_FILEDISTRIBUTOR -- ${ROOT}/sbin/filedistributor --configid $VESPA_CONFIG_ID +fi diff --git a/configserver/src/main/sh/start-logd b/configserver/src/main/sh/start-logd new file mode 100644 index 00000000000..4a797c1ce5c --- /dev/null +++ b/configserver/src/main/sh/start-logd @@ -0,0 +1,72 @@ +#! /bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME + +export VESPA_CONFIG_ID="file:$VESPA_HOME/conf/logd/logd.cfg" + +if [ "$multitenant" = "true" ]; then + PIDFILE_LOGD=var/run/logd.pid + VESPA_SERVICE_NAME=logd + export VESPA_SERVICE_NAME + vespa-runserver -r 30 -s logd -p $PIDFILE_LOGD -- ${ROOT}/sbin/logd +fi diff --git a/configserver/src/main/sh/stop-configserver b/configserver/src/main/sh/stop-configserver new file mode 100755 index 00000000000..1d63a25fa0e --- /dev/null +++ b/configserver/src/main/sh/stop-configserver @@ -0,0 +1,94 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=$VESPA_HOME +export ROOT +cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } + +# get common PATH etc: +. $ROOT/libexec/vespa/common-env.sh + +fixlimits + +# runserver takes care of making sure that we're not running several +# instances and saving its pid in this file: +PIDFILE_CONFIGSERVER=${ROOT}/var/run/configserver.pid +PIDFILE_FILEDISTRIBUTOR=var/run/filedistributor.pid +PIDFILE_LOGD=var/run/logd.pid + +VESPA_LOG_TARGET="file:${ROOT}/logs/vespa/vespa.log" +export VESPA_LOG_TARGET + +multitenant=$cloudconfig_server__multitenant +if [ "$multitenant" = "true" ]; then + run-as-yahoo vespa-runserver -s filedistributor -p $PIDFILE_FILEDISTRIBUTOR -S + run-as-yahoo vespa-runserver -s logd -p $PIDFILE_LOGD -S +fi + +# Try shutting down this way in case of upgrade. Can be removed in later versions. +run-as-yahoo vespa-runserver -s configserver -p $PIDFILE_CONFIGSERVER -S + +if [ -e "$PIDFILE_CONFIGSERVER" ]; then + export UNPRIVILEGED=1 + export PID_FILE=$PIDFILE_CONFIGSERVER + exec run-as-yahoo ${ROOT}/bin/jdisc_container_stop +fi diff --git a/configserver/src/test/apps/app/hosts.xml b/configserver/src/test/apps/app/hosts.xml new file mode 100644 index 00000000000..49e2450b69c --- /dev/null +++ b/configserver/src/test/apps/app/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="mytesthost"> + <alias>node1</alias> + </host> +</hosts> diff --git a/configserver/src/test/apps/app/searchdefinitions/music.sd b/configserver/src/test/apps/app/searchdefinitions/music.sd new file mode 100644 index 00000000000..891590f6f39 --- /dev/null +++ b/configserver/src/test/apps/app/searchdefinitions/music.sd @@ -0,0 +1,57 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# A basic search definition - called music, should be saved to music.sd +search music { + + # It contains one document type only - called music as well + document music { + + field title type string { + indexing: summary | index # How this field should be indexed + # index-to: title, default # Create two indexes + weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature + header + } + + field artist type string { + indexing: summary | attribute | index + # index-to: artist, default + + weight: 25 + header + } + + field year type int { + indexing: summary | attribute + header + } + + # Increase query + field popularity type int { + indexing: summary | attribute + body + } + + field url type uri { + indexing: summary | index + header + } + + } + + rank-profile default inherits default { + first-phase { + expression: nativeRank(title,artist) + attribute(popularity) + } + + } + + rank-profile textmatch inherits default { + first-phase { + expression: nativeRank(title,artist) + } + + } + + + +} diff --git a/configserver/src/test/apps/app/services.xml b/configserver/src/test/apps/app/services.xml new file mode 100644 index 00000000000..47fb12d5ebb --- /dev/null +++ b/configserver/src/test/apps/app/services.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + + <content version="1.0"> + <redundancy>2</redundancy> + <documents> + <document type="music" mode="index"/> + </documents> + <nodes> + <node hostalias="node1" distribution-key="0"/> + </nodes> + + </content> + + <jdisc version="1.0"> + <document-processing compressdocuments="true"> + <chain id="ContainerWrapperTest"> + <documentprocessor id="com.yahoo.vespa.config.AppleDocProc"/> + </chain> + </document-processing> + + <config name="project.specific"> + <value>someval</value> + </config> + + <nodes> + <node hostalias="node1" /> + </nodes> + + </jdisc> + +</services> diff --git a/configserver/src/test/apps/app_sdbundles/components/testbundle.jar b/configserver/src/test/apps/app_sdbundles/components/testbundle.jar Binary files differnew file mode 100644 index 00000000000..00749d776c2 --- /dev/null +++ b/configserver/src/test/apps/app_sdbundles/components/testbundle.jar diff --git a/configserver/src/test/apps/app_sdbundles/components/testbundle2.jar b/configserver/src/test/apps/app_sdbundles/components/testbundle2.jar Binary files differnew file mode 100644 index 00000000000..36c97c2716c --- /dev/null +++ b/configserver/src/test/apps/app_sdbundles/components/testbundle2.jar diff --git a/configserver/src/test/apps/app_sdbundles/files/foo.txt b/configserver/src/test/apps/app_sdbundles/files/foo.txt new file mode 100644 index 00000000000..b7d6715e2df --- /dev/null +++ b/configserver/src/test/apps/app_sdbundles/files/foo.txt @@ -0,0 +1 @@ +FOO diff --git a/configserver/src/test/apps/app_sdbundles/files/subdir/bar.txt b/configserver/src/test/apps/app_sdbundles/files/subdir/bar.txt new file mode 100644 index 00000000000..ba578e48b18 --- /dev/null +++ b/configserver/src/test/apps/app_sdbundles/files/subdir/bar.txt @@ -0,0 +1 @@ +BAR diff --git a/configserver/src/test/apps/app_sdbundles/hosts.xml b/configserver/src/test/apps/app_sdbundles/hosts.xml new file mode 100644 index 00000000000..fc545b34f6f --- /dev/null +++ b/configserver/src/test/apps/app_sdbundles/hosts.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> + <host name="schmocalhost"> + <alias>node2</alias> + </host> +</hosts> + diff --git a/configserver/src/test/apps/app_sdbundles/services.xml b/configserver/src/test/apps/app_sdbundles/services.xml new file mode 100644 index 00000000000..2e5e13cc6b0 --- /dev/null +++ b/configserver/src/test/apps/app_sdbundles/services.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + <slobroks> + <slobrok hostalias="node1"/> + <slobrok hostalias="node2"/> + </slobroks> + </admin> + + <content version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index"/> + </documents> + <nodes>> + <node hostalias="node1" distribution-key="0"/> + </nodes> + </content> + +</services> diff --git a/configserver/src/test/apps/components/com.yahoo.searcher1.jar b/configserver/src/test/apps/components/com.yahoo.searcher1.jar Binary files differnew file mode 100644 index 00000000000..437246152db --- /dev/null +++ b/configserver/src/test/apps/components/com.yahoo.searcher1.jar diff --git a/configserver/src/test/apps/content/.ignored b/configserver/src/test/apps/content/.ignored new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/configserver/src/test/apps/content/.ignored diff --git a/configserver/src/test/apps/content/foo/.ignored b/configserver/src/test/apps/content/foo/.ignored new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/configserver/src/test/apps/content/foo/.ignored diff --git a/configserver/src/test/apps/content/foo/bar/test.txt b/configserver/src/test/apps/content/foo/bar/test.txt new file mode 100644 index 00000000000..401a9f6e542 --- /dev/null +++ b/configserver/src/test/apps/content/foo/bar/test.txt @@ -0,0 +1 @@ +bim diff --git a/configserver/src/test/apps/content/foo/test1.txt b/configserver/src/test/apps/content/foo/test1.txt new file mode 100644 index 00000000000..5716ca5987c --- /dev/null +++ b/configserver/src/test/apps/content/foo/test1.txt @@ -0,0 +1 @@ +bar diff --git a/configserver/src/test/apps/content/foo/test2.txt b/configserver/src/test/apps/content/foo/test2.txt new file mode 100644 index 00000000000..76018072e09 --- /dev/null +++ b/configserver/src/test/apps/content/foo/test2.txt @@ -0,0 +1 @@ +baz diff --git a/configserver/src/test/apps/content/newtest/testfile.txt b/configserver/src/test/apps/content/newtest/testfile.txt new file mode 100644 index 00000000000..10836e6a1e6 --- /dev/null +++ b/configserver/src/test/apps/content/newtest/testfile.txt @@ -0,0 +1 @@ +bario diff --git a/configserver/src/test/apps/content/test.txt b/configserver/src/test/apps/content/test.txt new file mode 100644 index 00000000000..257cc5642cb --- /dev/null +++ b/configserver/src/test/apps/content/test.txt @@ -0,0 +1 @@ +foo diff --git a/configserver/src/test/apps/content2/test.txt b/configserver/src/test/apps/content2/test.txt new file mode 100644 index 00000000000..5716ca5987c --- /dev/null +++ b/configserver/src/test/apps/content2/test.txt @@ -0,0 +1 @@ +bar diff --git a/configserver/src/test/apps/cs1/hosts.xml b/configserver/src/test/apps/cs1/hosts.xml new file mode 100644 index 00000000000..bce37f6facc --- /dev/null +++ b/configserver/src/test/apps/cs1/hosts.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="mytesthost"> + <alias>node0</alias> + </host> +</hosts> diff --git a/configserver/src/test/apps/cs1/services.xml b/configserver/src/test/apps/cs1/services.xml new file mode 100644 index 00000000000..cc5c59dac09 --- /dev/null +++ b/configserver/src/test/apps/cs1/services.xml @@ -0,0 +1,9 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + <config name="config.simpletypes"> + <intval>1337</intval> + </config> + <admin version="2.0"> + <adminserver hostalias="node0" /> + </admin> +</services> diff --git a/configserver/src/test/apps/cs2/hosts.xml b/configserver/src/test/apps/cs2/hosts.xml new file mode 100644 index 00000000000..9f7d8ed360a --- /dev/null +++ b/configserver/src/test/apps/cs2/hosts.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="mytesthost2"> + <alias>node0</alias> + </host> +</hosts> diff --git a/configserver/src/test/apps/cs2/services.xml b/configserver/src/test/apps/cs2/services.xml new file mode 100644 index 00000000000..729f9f3f43c --- /dev/null +++ b/configserver/src/test/apps/cs2/services.xml @@ -0,0 +1,9 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + <config name="config.simpletypes"> + <intval>1330</intval> + </config> + <admin version="2.0"> + <adminserver hostalias="node0" /> + </admin> +</services> diff --git a/configserver/src/test/apps/hosted/searchdefinitions/music.sd b/configserver/src/test/apps/hosted/searchdefinitions/music.sd new file mode 100644 index 00000000000..4e5f4a60275 --- /dev/null +++ b/configserver/src/test/apps/hosted/searchdefinitions/music.sd @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music { + field title type string { + indexing: index | summary + # index-to: default + } + } +} + diff --git a/configserver/src/test/apps/hosted/services.xml b/configserver/src/test/apps/hosted/services.xml new file mode 100644 index 00000000000..3f70580f8e2 --- /dev/null +++ b/configserver/src/test/apps/hosted/services.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="3.0"> + <nodes count='1'/> + </admin> + + <jdisc version="1.0"> + <search/> + <nodes count='1'/> + </jdisc> + + <content id="music" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index" /> + </documents> + <nodes count="2" groups="2"/> + </content> + +</services> diff --git a/configserver/src/test/apps/illegalApp/services.xml b/configserver/src/test/apps/illegalApp/services.xml new file mode 100644 index 00000000000..9dd45ebc84a --- /dev/null +++ b/configserver/src/test/apps/illegalApp/services.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + <admin version="2.0"> + <adminserver /> + <unknownelement /> + </admin> +</services> diff --git a/configserver/src/test/apps/legalApp/services.xml b/configserver/src/test/apps/legalApp/services.xml new file mode 100644 index 00000000000..816fae9cc95 --- /dev/null +++ b/configserver/src/test/apps/legalApp/services.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + <admin version="2.0"> + <adminserver /> + </admin> +</services> diff --git a/configserver/src/test/apps/serverdb/serverdefs/attributes.def b/configserver/src/test/apps/serverdb/serverdefs/attributes.def new file mode 100644 index 00000000000..ae8ff92d48b --- /dev/null +++ b/configserver/src/test/apps/serverdb/serverdefs/attributes.def @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=config +attribute[].name string +attribute[].datatype string +attribute[].multivalue bool default=false +attribute[].sortsigned bool default=true +attribute[].disableprep bool default=false diff --git a/configserver/src/test/apps/zkapp/components/defs-only.jar b/configserver/src/test/apps/zkapp/components/defs-only.jar Binary files differnew file mode 100644 index 00000000000..681301a3d8b --- /dev/null +++ b/configserver/src/test/apps/zkapp/components/defs-only.jar diff --git a/configserver/src/test/apps/zkapp/components/file.txt b/configserver/src/test/apps/zkapp/components/file.txt new file mode 100644 index 00000000000..e167ca380f5 --- /dev/null +++ b/configserver/src/test/apps/zkapp/components/file.txt @@ -0,0 +1 @@ +/home/vespa/test/file.txt
\ No newline at end of file diff --git a/configserver/src/test/apps/zkapp/deployment.xml b/configserver/src/test/apps/zkapp/deployment.xml new file mode 100644 index 00000000000..a9e9fdff07e --- /dev/null +++ b/configserver/src/test/apps/zkapp/deployment.xml @@ -0,0 +1,8 @@ +<?xml version='1.0' encoding='UTF-8'?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<deployment version='1.0'> + <test/> + <prod global-service-id='mydisc'>+ + <region active='true'>us-east</region> + </prod> +</deployment> diff --git a/configserver/src/test/apps/zkapp/files/foo.json b/configserver/src/test/apps/zkapp/files/foo.json new file mode 100644 index 00000000000..ed72b09660a --- /dev/null +++ b/configserver/src/test/apps/zkapp/files/foo.json @@ -0,0 +1 @@ +foo : foo diff --git a/configserver/src/test/apps/zkapp/files/sub/bar.json b/configserver/src/test/apps/zkapp/files/sub/bar.json new file mode 100644 index 00000000000..2f008f410ec --- /dev/null +++ b/configserver/src/test/apps/zkapp/files/sub/bar.json @@ -0,0 +1 @@ +bar : bar diff --git a/configserver/src/test/apps/zkapp/hosts.xml b/configserver/src/test/apps/zkapp/hosts.xml new file mode 100644 index 00000000000..fc545b34f6f --- /dev/null +++ b/configserver/src/test/apps/zkapp/hosts.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> + <host name="schmocalhost"> + <alias>node2</alias> + </host> +</hosts> + diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression b/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression new file mode 100644 index 00000000000..eed496e6aeb --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression @@ -0,0 +1 @@ +bar(f*2) diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression b/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression new file mode 100644 index 00000000000..ce26aa75dcb --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression @@ -0,0 +1 @@ +foo()+1 diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd b/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd new file mode 100644 index 00000000000..147e128df16 --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd @@ -0,0 +1,41 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search laptop { + + document laptop inherits product { + + field batterycapacity type int { + indexing: attribute + } + + field location_str type array<string> { + + } + } + + field batteryrank type int { + indexing: input batterycapacity | attribute + } + + field location type array<position> { + indexing: input location_str | for_each { to_pos } | attribute + } + + rank-profile default { + second-phase { + expression: fieldMatch(title)*fieldMatch(title).weight + rerank-count: 150 + } + summary-features: fieldMatch(title) + + rank-features: attribute(batterycapacity) match.weight.batterycapacity + + rank-properties { + fieldMatch(title).maxOccurrences : 40 + fieldMatch(title).proximityLimit : 5 + } + } + + rank-profile batteryranked { + } + +} diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/music.sd b/configserver/src/test/apps/zkapp/searchdefinitions/music.sd new file mode 100644 index 00000000000..d0eec200b90 --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/music.sd @@ -0,0 +1,44 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# A basic search definition - called music, should be saved to music.sd +search music { + + # It contains one document type only - called music as well + document music { + + field title type string { + indexing: summary | index # How this field should be indexed + # index-to: title, default # Create two indexes + rank-type: about # Type of ranking settings to apply + header + } + + field artist type string { + indexing: summary | attribute | index + # index-to: artist, default + rank-type:about + header + } + + field year type int { + indexing: summary | attribute + header + } + + # Increase rank score of popular documents regardless of query + field popularity type int { + indexing: summary | attribute + body + } + + field url type uri { + indexing: summary | index + header + } + + field cover type raw { + body + } + + } + +} diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd b/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd new file mode 100644 index 00000000000..89f9ffe530d --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd @@ -0,0 +1,47 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search pc { + + document pc inherits product { + + field brand type string { + indexing: index | summary + } + + field color type string { + indexing: summary | index + index: prefix + alias: colour + rank: filter + } + + field cpuspeed type int { + indexing: summary + } + + field location_str type array<string> { + + } + } + + field location type array<position> { + indexing: input location_str | for_each { to_pos } | attribute + } + + rank-profile default { + first-phase { + expression: fieldMatch(brand).completeness + fieldMatch(color).completeness + } + second-phase { + expression: fieldMatch(brand).completeness*fieldMatch(brand).importancy + fieldMatch(color).completeness*fieldMatch(color).importancy + } + + summary-features: fieldMatch(title) fieldMatch(brand).proximity match.weight.title nativeFieldMatch(title) + + rank-features: attribute(cpuspeed) + + rank-properties { + fieldMatch(brand).maxOccurrences : 20 + } + } + +} diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/product.sd b/configserver/src/test/apps/zkapp/searchdefinitions/product.sd new file mode 100644 index 00000000000..d8b1d725d1c --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/product.sd @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +document product { + + field title type string { + indexing: index | summary + # index-to: title, default + } + + field price type int { + indexing: index | summary | attribute + } + +} diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd b/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd new file mode 100644 index 00000000000..1620d790b65 --- /dev/null +++ b/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd @@ -0,0 +1,27 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search sock { + + document sock inherits product { + + field size type int { + indexing: index | summary | attribute + } + + field color type string { + indexing: summary + index: prefix + } + + field brand type string { + indexing: summary + } + + } + + rank-profile other { + second-phase { + expression: fieldMatch(color).fieldCompleteness + fieldMatch(brand).proximity + } + } + +} diff --git a/configserver/src/test/apps/zkapp/services.xml b/configserver/src/test/apps/zkapp/services.xml new file mode 100644 index 00000000000..aee18cc450a --- /dev/null +++ b/configserver/src/test/apps/zkapp/services.xml @@ -0,0 +1,60 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + <slobroks> + <slobrok hostalias="node1"/> + <slobrok hostalias="node2"/> + </slobroks> + </admin> + + <clients version="2.0"> + <gateways protocols="http"> + <gateway hostalias="node1" /> + </gateways> + </clients> + + <search version="2.0"> + <qrservers> + <qrserver hostalias="node1" /> + </qrservers> + <cluster name="music" indexingmode="realtime"> + <searchdefinitions> + <searchdefinition name="music" /> + </searchdefinitions> + <clustercontrollers> + <clustercontroller hostalias="node1" /> + </clustercontrollers> + <topleveldispatchers> + <topleveldispatcher hostalias="node1" /> + </topleveldispatchers> + <row index="0"> + <searchnodes> + <searchnode hostalias="node1" index="0" /> + </searchnodes> + </row> + </cluster> + </search> + + <storage version="3.0"> + <cluster redundancy="1"> +<!-- +Do not reshuffle nodes or change index values - this will cause +massive document redistribution. + +If you want to discontinue use of a node, set it in the 'retired' state, +this will rebalance the documents out of the node. Once node is empty, +you can stop it and delete the reference to it in vespa-services and +vespa-hosts files. +--> + <group index="0" name="mycluster"> + <node hostalias="node1" index="0"/> + </group> + + <fleetcontroller hostalias="node1" /> + </cluster> + </storage> + +</services> diff --git a/configserver/src/test/apps/zkfeed/.gitignore b/configserver/src/test/apps/zkfeed/.gitignore new file mode 100644 index 00000000000..d35471b1021 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/.gitignore @@ -0,0 +1 @@ +.applicationMetaData diff --git a/configserver/src/test/apps/zkfeed/components/defs-only.jar b/configserver/src/test/apps/zkfeed/components/defs-only.jar Binary files differnew file mode 100644 index 00000000000..28f563cd779 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/components/defs-only.jar diff --git a/configserver/src/test/apps/zkfeed/components/file.txt b/configserver/src/test/apps/zkfeed/components/file.txt new file mode 100644 index 00000000000..e167ca380f5 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/components/file.txt @@ -0,0 +1 @@ +/home/vespa/test/file.txt
\ No newline at end of file diff --git a/configserver/src/test/apps/zkfeed/components/testbundle.jar b/configserver/src/test/apps/zkfeed/components/testbundle.jar Binary files differnew file mode 100644 index 00000000000..00749d776c2 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/components/testbundle.jar diff --git a/configserver/src/test/apps/zkfeed/configs/fdispatchrc.cfg b/configserver/src/test/apps/zkfeed/configs/fdispatchrc.cfg new file mode 100644 index 00000000000..257cc5642cb --- /dev/null +++ b/configserver/src/test/apps/zkfeed/configs/fdispatchrc.cfg @@ -0,0 +1 @@ +foo diff --git a/configserver/src/test/apps/zkfeed/dir1/default.xml b/configserver/src/test/apps/zkfeed/dir1/default.xml new file mode 100644 index 00000000000..f1e16333fc1 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/dir1/default.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<search> + <chain id="default"> + <searcher id="com.yahoo.search.example.SimpleSearcher" bundle="mybundle"/> + </chain> +</search> diff --git a/configserver/src/test/apps/zkfeed/files/foo.json b/configserver/src/test/apps/zkfeed/files/foo.json new file mode 100644 index 00000000000..ed72b09660a --- /dev/null +++ b/configserver/src/test/apps/zkfeed/files/foo.json @@ -0,0 +1 @@ +foo : foo diff --git a/configserver/src/test/apps/zkfeed/files/sub/bar.json b/configserver/src/test/apps/zkfeed/files/sub/bar.json new file mode 100644 index 00000000000..2f008f410ec --- /dev/null +++ b/configserver/src/test/apps/zkfeed/files/sub/bar.json @@ -0,0 +1 @@ +bar : bar diff --git a/configserver/src/test/apps/zkfeed/hosts.xml b/configserver/src/test/apps/zkfeed/hosts.xml new file mode 100644 index 00000000000..fc545b34f6f --- /dev/null +++ b/configserver/src/test/apps/zkfeed/hosts.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> + <host name="schmocalhost"> + <alias>node2</alias> + </host> +</hosts> + diff --git a/configserver/src/test/apps/zkfeed/nested/dir2/chain2.xml b/configserver/src/test/apps/zkfeed/nested/dir2/chain2.xml new file mode 100644 index 00000000000..9d297be5212 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/nested/dir2/chain2.xml @@ -0,0 +1,8 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<search> + <searcher class="com.yahoo.search.example.SimpleSearcher" id="s1" bundle="mybundle"/> + <chain id="chain2"> + <searcher id="s1"/> + <searcher id="com.yahoo.search.example.SimpleSearcher2" bundle="mybundle"/> + </chain> +</search> diff --git a/configserver/src/test/apps/zkfeed/nested/dir2/chain3.xml b/configserver/src/test/apps/zkfeed/nested/dir2/chain3.xml new file mode 100644 index 00000000000..0e019ba9d02 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/nested/dir2/chain3.xml @@ -0,0 +1,10 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<search> + <chain id="chain3_1"> + <searcher id="com.yahoo.search.example.SimpleSearcher" bundle="mybundle"/> + </chain> + <chain id="chain3_2"> + <searcher id="com.yahoo.search.example.SimpleSearcher" bundle="mybundle"/> + <searcher id="com.yahoo.search.example.SimpleSearcher2" bundle="mybundle"/> + </chain> +</search> diff --git a/configserver/src/test/apps/zkfeed/search/chains/dir1/default.xml b/configserver/src/test/apps/zkfeed/search/chains/dir1/default.xml new file mode 100644 index 00000000000..0872d66c385 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/search/chains/dir1/default.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<searchchains> +<searchchain id="default"> + <searcher id="com.yahoo.search.example.SimpleSearcher" bundle="mybundle"/> +</searchchain> +</searchchains> diff --git a/configserver/src/test/apps/zkfeed/search/chains/dir2/chain2.xml b/configserver/src/test/apps/zkfeed/search/chains/dir2/chain2.xml new file mode 100644 index 00000000000..a1405257192 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/search/chains/dir2/chain2.xml @@ -0,0 +1,8 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<searchchains> +<searcher class="com.yahoo.search.example.SimpleSearcher" id="s1" bundle="mybundle"/> +<searchchain id="chain2"> + <searcher id="s1"/> + <searcher id="com.yahoo.search.example.SimpleSearcher2" bundle="mybundle"/> +</searchchain> +</searchchains> diff --git a/configserver/src/test/apps/zkfeed/search/chains/dir2/chain3.xml b/configserver/src/test/apps/zkfeed/search/chains/dir2/chain3.xml new file mode 100644 index 00000000000..138db126b19 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/search/chains/dir2/chain3.xml @@ -0,0 +1,10 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<searchchains> +<searchchain id="chain3_1"> + <searcher id="com.yahoo.search.example.SimpleSearcher" bundle="mybundle"/> +</searchchain> +<searchchain id="chain3_2"> + <searcher id="com.yahoo.search.example.SimpleSearcher" bundle="mybundle"/> + <searcher id="com.yahoo.search.example.SimpleSearcher2" bundle="mybundle"/> +</searchchain> +</searchchains> diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression b/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression new file mode 100644 index 00000000000..eed496e6aeb --- /dev/null +++ b/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression @@ -0,0 +1 @@ +bar(f*2) diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression b/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression new file mode 100644 index 00000000000..ce26aa75dcb --- /dev/null +++ b/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression @@ -0,0 +1 @@ +foo()+1 diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd b/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd new file mode 100644 index 00000000000..147e128df16 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd @@ -0,0 +1,41 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search laptop { + + document laptop inherits product { + + field batterycapacity type int { + indexing: attribute + } + + field location_str type array<string> { + + } + } + + field batteryrank type int { + indexing: input batterycapacity | attribute + } + + field location type array<position> { + indexing: input location_str | for_each { to_pos } | attribute + } + + rank-profile default { + second-phase { + expression: fieldMatch(title)*fieldMatch(title).weight + rerank-count: 150 + } + summary-features: fieldMatch(title) + + rank-features: attribute(batterycapacity) match.weight.batterycapacity + + rank-properties { + fieldMatch(title).maxOccurrences : 40 + fieldMatch(title).proximityLimit : 5 + } + } + + rank-profile batteryranked { + } + +} diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd b/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd new file mode 100644 index 00000000000..89f9ffe530d --- /dev/null +++ b/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd @@ -0,0 +1,47 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search pc { + + document pc inherits product { + + field brand type string { + indexing: index | summary + } + + field color type string { + indexing: summary | index + index: prefix + alias: colour + rank: filter + } + + field cpuspeed type int { + indexing: summary + } + + field location_str type array<string> { + + } + } + + field location type array<position> { + indexing: input location_str | for_each { to_pos } | attribute + } + + rank-profile default { + first-phase { + expression: fieldMatch(brand).completeness + fieldMatch(color).completeness + } + second-phase { + expression: fieldMatch(brand).completeness*fieldMatch(brand).importancy + fieldMatch(color).completeness*fieldMatch(color).importancy + } + + summary-features: fieldMatch(title) fieldMatch(brand).proximity match.weight.title nativeFieldMatch(title) + + rank-features: attribute(cpuspeed) + + rank-properties { + fieldMatch(brand).maxOccurrences : 20 + } + } + +} diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd b/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd new file mode 100644 index 00000000000..d8b1d725d1c --- /dev/null +++ b/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +document product { + + field title type string { + indexing: index | summary + # index-to: title, default + } + + field price type int { + indexing: index | summary | attribute + } + +} diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd b/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd new file mode 100644 index 00000000000..1620d790b65 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd @@ -0,0 +1,27 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search sock { + + document sock inherits product { + + field size type int { + indexing: index | summary | attribute + } + + field color type string { + indexing: summary + index: prefix + } + + field brand type string { + indexing: summary + } + + } + + rank-profile other { + second-phase { + expression: fieldMatch(color).fieldCompleteness + fieldMatch(brand).proximity + } + } + +} diff --git a/configserver/src/test/apps/zkfeed/services.xml b/configserver/src/test/apps/zkfeed/services.xml new file mode 100644 index 00000000000..ecde7dfade8 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/services.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1" /> + <logserver hostalias="node1" /> + </admin> + + <jdisc version="1.0"> + <search> + <include dir='dir1'/> + <include dir='nested/dir2'/> + </search> + + <nodes> + <node hostalias="node1" /> + </nodes> + </jdisc> + +</services> diff --git a/configserver/src/test/apps/zkfeed/templates/basic/error.templ b/configserver/src/test/apps/zkfeed/templates/basic/error.templ new file mode 100644 index 00000000000..87c3eb12f3c --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/basic/error.templ @@ -0,0 +1 @@ +<error code="$result.error.code">$result.error.message</error> diff --git a/configserver/src/test/apps/zkfeed/templates/basic/header.templ b/configserver/src/test/apps/zkfeed/templates/basic/header.templ new file mode 100644 index 00000000000..2a24f8c0e2c --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/basic/header.templ @@ -0,0 +1,2 @@ +<?xml version="1.0" encoding="utf-8" ?> +<resultset totalhits="$result.hitCount"> diff --git a/configserver/src/test/apps/zkfeed/templates/basic/hit.templ b/configserver/src/test/apps/zkfeed/templates/basic/hit.templ new file mode 100644 index 00000000000..b25c43af3f5 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/basic/hit.templ @@ -0,0 +1,5 @@ +<hit relevancy="$relevancy"> +#foreach( $key in $hit.getPropertyKeySet() ) + <field name='$key'>$hit.getPropertyXML($key)</field> +#end +</hit> diff --git a/configserver/src/test/apps/zkfeed/templates/simple_html/footer.templ b/configserver/src/test/apps/zkfeed/templates/simple_html/footer.templ new file mode 100644 index 00000000000..79b660983a9 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/simple_html/footer.templ @@ -0,0 +1,4 @@ +<HR> +<FONT SIZE=-1>Yahoo! Vespa</FONT> +</BODY> +</HTML> diff --git a/configserver/src/test/apps/zkfeed/templates/simple_html/header.templ b/configserver/src/test/apps/zkfeed/templates/simple_html/header.templ new file mode 100644 index 00000000000..7e1c6a1df68 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/simple_html/header.templ @@ -0,0 +1,21 @@ +<HTML> +<HEAD> +<TITLE>Search results</TITLE> +</HEAD> +<BODY BGCOLOR="#FFFFFF" TEXT="#000000"> + +<form action=$result.query.path name="search"> +<input TYPE="text" SIZE="35" NAME="query" VALUE=""> +<input TYPE=submit VALUE="Search"> +</form> + +<script> +<!-- +function setfocus() { document.search.query.focus(); } setfocus(); +// --> +</script> + +<H3>Search for "$result.query.rawQueryHTMLEncoded"</H3> +$result.totalHitCount hits +<BR><HR><BR> +<!--<A HREF="/">Home</A><BR><BR>--> diff --git a/configserver/src/test/apps/zkfeed/templates/simple_html/hit.templ b/configserver/src/test/apps/zkfeed/templates/simple_html/hit.templ new file mode 100644 index 00000000000..2ea10c7bc76 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/simple_html/hit.templ @@ -0,0 +1,4 @@ +<B>$hitno</B> ($relevancy) <A HREF="$uri"><B>$title</B></A><BR> +$artist - $title ($year)<BR> +<FONT SIZE=-1 COLOR=GREEN>($uri)</FONT><BR> +<BR> diff --git a/configserver/src/test/apps/zkfeed/templates/text/error.templ b/configserver/src/test/apps/zkfeed/templates/text/error.templ new file mode 100644 index 00000000000..7e41bee9e65 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/text/error.templ @@ -0,0 +1,4 @@ +ERROR ------------------------------------------------------------------ + result.error.code = '$result.error.code' + result.error.message = '$result.error.message' +------------------------------------------------------------------------ diff --git a/configserver/src/test/apps/zkfeed/templates/text/footer.templ b/configserver/src/test/apps/zkfeed/templates/text/footer.templ new file mode 100644 index 00000000000..dfc9c240a83 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/text/footer.templ @@ -0,0 +1,2 @@ +FOOTER ----------------------------------------------------------------- +------------------------------------------------------------------------ diff --git a/configserver/src/test/apps/zkfeed/templates/text/header.templ b/configserver/src/test/apps/zkfeed/templates/text/header.templ new file mode 100644 index 00000000000..ce62e70470e --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/text/header.templ @@ -0,0 +1,17 @@ +HEADER ----------------------------------------------------------------- +result.hitCount = '$result.hitCount' +result.query = '$result.query' +result.query.path = '$result.query.path' +result.query.rawQuery = '$result.query.rawQuery' +result.query.rawQueryHTMLEncoded = '$result.query.rawQueryHTMLEncoded' +result.query.rawQueryURLEncoded = '$result.query.rawQueryURLEncoded' +result.query.hits = '$result.query.hits' +result.query.queryType = '$result.query.queryType' +result.firstHitNo = '$result.firstHitNo' +result.totalHitCount = '$result.totalHitCount' +result.totalSearchTime = '$result.totalSearchTime' +result.prevFirstHitNo = '$result.prevFirstHitNo' +result.prevLastHitNo = '$result.prevLastHitNo' +result.nextResultURL = '$result.nextResultURL' +result.previousResultURL = '$result.previousResultURL' +------------------------------------------------------------------------ diff --git a/configserver/src/test/apps/zkfeed/templates/text/hit.templ b/configserver/src/test/apps/zkfeed/templates/text/hit.templ new file mode 100644 index 00000000000..eeefbabf607 --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/text/hit.templ @@ -0,0 +1,15 @@ +HIT -------------------------------------------------------------------- +uri = '$uri' +relevancy = '$relevancy' +hitno = '$hitno' +hit.typeString = '$hit.typeString' +category = '$category' +bsumtitle = '$bsumtitle' +hit.siteId = '$hit.siteId' +hit.source = '$hit.source' + +#foreach( $key in $hit.getPropertyKeySet() ) +'$key'='$hit.getProperty($key)' +#end + +------------------------------------------------------------------------ diff --git a/configserver/src/test/apps/zkfeed/templates/text/nohits.templ b/configserver/src/test/apps/zkfeed/templates/text/nohits.templ new file mode 100644 index 00000000000..f1b12d5c21a --- /dev/null +++ b/configserver/src/test/apps/zkfeed/templates/text/nohits.templ @@ -0,0 +1,3 @@ +NOHITS ----------------------------------------------------------------- +(empty) +------------------------------------------------------------------------ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationMapperTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationMapperTest.java new file mode 100644 index 00000000000..e5d98bda7b5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationMapperTest.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Optional; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.http.NotFoundException; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ApplicationMapperTest { + ApplicationId appId; + ApplicationMapper applicationMapper; + ArrayList<Version> vespaVersions = new ArrayList<>(); + ArrayList<Application> applications = new ArrayList<>(); + + @Before + public void setUp() { + applicationMapper = new ApplicationMapper(); + appId = new ApplicationId.Builder() + .tenant("test").applicationName("test").instanceName("test").build(); + vespaVersions.add(Version.fromString("1.2.3")); + vespaVersions.add(Version.fromString("1.2.4")); + vespaVersions.add(Version.fromString("1.2.5")); + applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(0), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(1), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(2), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + } + + @Test + public void testGetForVersionReturnsCorrectVersion() { + applicationMapper.register(appId, ApplicationSet.fromList(applications)); + assertEquals(applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(0))), applications.get(0)); + assertEquals(applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(1))), applications.get(1)); + assertEquals(applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(2))), applications.get(2)); + } + + @Test + public void testGetForVersionReturnsLatestVersion() { + applicationMapper.register(appId, ApplicationSet.fromList(applications)); + assertEquals(applicationMapper.getForVersion(appId, Optional.empty()), applications.get(2)); + } + + @Test (expected = VersionDoesNotExistException.class) + public void testGetForVersionThrows() { + applicationMapper.register(appId, ApplicationSet.fromList(Arrays.asList(applications.get(0), applications.get(2)))); + + applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(1))); + } + + @Test (expected = NotFoundException.class) + public void testGetForVersionThrows2() { + applicationMapper.register(appId, ApplicationSet.fromSingle(applications.get(0))); + + applicationMapper.getForVersion(new ApplicationId.Builder() + .tenant("different").applicationName("different").instanceName("different").build(), + Optional.of(vespaVersions.get(1))); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationSetTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationSetTest.java new file mode 100644 index 00000000000..fce6139e18e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationSetTest.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author Vegard Sjonfjell + */ +public class ApplicationSetTest { + + ApplicationSet applicationSet; + List<Version> vespaVersions = new ArrayList<>(); + List<Application> applications = new ArrayList<>(); + + @Before + public void setUp() { + vespaVersions.add(Version.fromString("1.2.3")); + vespaVersions.add(Version.fromString("1.2.4")); + vespaVersions.add(Version.fromString("1.2.5")); + applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(0), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(1), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + applications.add(new Application(new ModelStub(), null, 0, vespaVersions.get(2), MetricUpdater.createTestUpdater(), ApplicationId.defaultId())); + } + + @Test + public void testGetForVersionOrLatestReturnsCorrectVersion() { + applicationSet = ApplicationSet.fromList(applications); + assertEquals(applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(0))), applications.get(0)); + assertEquals(applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(1))), applications.get(1)); + assertEquals(applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(2))), applications.get(2)); + } + + @Test + public void testGetForVersionOrLatestReturnsLatestVersion() { + applicationSet = ApplicationSet.fromList(applications); + assertEquals(applicationSet.getForVersionOrLatest(Optional.empty()), applications.get(2)); + } + + @Test (expected = VersionDoesNotExistException.class) + public void testGetForVersionOrLatestThrows() { + applicationSet = ApplicationSet.fromList(Arrays.asList(applications.get(0), applications.get(2))); + applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(1))); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/CompressedApplicationInputStreamTest.java new file mode 100644 index 00000000000..5dd0f889431 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/CompressedApplicationInputStreamTest.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.io.ByteStreams; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.Test; + +import java.io.*; +import java.util.Arrays; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author lulf + * @since 5.1 + */ +public class CompressedApplicationInputStreamTest { + + static void writeFileToTar(ArchiveOutputStream taos, File file) throws IOException { + taos.putArchiveEntry(taos.createArchiveEntry(file, file.getName())); + ByteStreams.copy(new FileInputStream(file), taos); + taos.closeArchiveEntry(); + } + + public static File createArchiveFile(ArchiveOutputStream taos, File outFile) throws IOException { + File app = new File("src/test/resources/deploy/validapp"); + writeFileToTar(taos, new File(app, "services.xml")); + writeFileToTar(taos, new File(app, "hosts.xml")); + taos.close(); + return outFile; + } + + public static File createTarFile() throws IOException { + File outFile = File.createTempFile("testapp", ".tar.gz"); + ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); + return createArchiveFile(archiveOutputStream, outFile); + } + + public static File createZipFile() throws IOException { + File outFile = File.createTempFile("testapp", ".tar.gz"); + ArchiveOutputStream archiveOutputStream = new ZipArchiveOutputStream(new FileOutputStream(outFile)); + return createArchiveFile(archiveOutputStream, outFile); + } + + void assertTestApp(File outApp) { + String [] files = outApp.list(); + assertThat(files.length, is(2)); + if ("hosts.xml".equals(files[0])) { + assertThat(files[1], is("services.xml")); + } else if ("hosts.xml".equals(files[1])) { + assertThat(files[0], is("services.xml")); + } else { + fail("Both services.xml and hosts.xml should be contained in the unpacked application"); + } + } + + @Test + public void require_that_valid_tar_application_can_be_unpacked() throws IOException { + File outFile = createTarFile(); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(outFile)))); + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } + + @Test + public void require_that_valid_tar_application_in_subdir_can_be_unpacked() throws IOException { + File outFile = File.createTempFile("testapp", ".tar.gz"); + ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); + + File app = new File("src/test/resources/deploy/validapp"); + + File file = new File(app, "services.xml"); + archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); + ByteStreams.copy(new FileInputStream(file), archiveOutputStream); + archiveOutputStream.closeArchiveEntry(); + file = new File(app, "hosts.xml"); + archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); + ByteStreams.copy(new FileInputStream(file), archiveOutputStream); + archiveOutputStream.closeArchiveEntry(); + + archiveOutputStream.close(); + + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream(new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(outFile)))); + File outApp = unpacked.decompress(); + assertThat(outApp.getName(), is("application")); // gets the name of the subdir + assertTestApp(outApp); + } + + @Test + public void require_that_valid_zip_application_can_be_unpacked() throws IOException { + File outFile = createZipFile(); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( + new ZipArchiveInputStream(new FileInputStream(outFile))); + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } + + @Test + public void require_that_gnu_tared_file_can_be_unpacked() throws IOException, InterruptedException { + File tmpTar = File.createTempFile("myapp", ".tar"); + Process p = new ProcessBuilder("tar", "-C", "src/test/resources/deploy/validapp", "--exclude=.svn", "-cvf", tmpTar.getAbsolutePath(), ".").start(); + p.waitFor(); + p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); + p.waitFor(); + File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + assertTrue(gzFile.exists()); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( + new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); + File outApp = unpacked.decompress(); + assertTestApp(outApp); + } + + @Test + public void require_that_nested_app_can_be_unpacked() throws IOException, InterruptedException { + File tmpTar = File.createTempFile("myapp", ".tar"); + Process p = new ProcessBuilder("tar", "-C", "src/test/resources/deploy/advancedapp", "--exclude=.svn", "-cvf", tmpTar.getAbsolutePath(), ".").start(); + p.waitFor(); + p = new ProcessBuilder("gzip", tmpTar.getAbsolutePath()).start(); + p.waitFor(); + File gzFile = new File(tmpTar.getAbsolutePath() + ".gz"); + assertTrue(gzFile.exists()); + CompressedApplicationInputStream unpacked = CompressedApplicationInputStream.createFromCompressedStream( + new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(gzFile)))); + File outApp = unpacked.decompress(); + List<File> files = Arrays.asList(outApp.listFiles()); + assertThat(files.size(), is(4)); + assertTrue(files.contains(new File(outApp, "services.xml"))); + assertTrue(files.contains(new File(outApp, "hosts.xml"))); + assertTrue(files.contains(new File(outApp, "searchdefinitions"))); + assertTrue(files.contains(new File(outApp, "external"))); + File sd = files.get(files.indexOf(new File(outApp, "searchdefinitions"))); + assertTrue(sd.isDirectory()); + assertThat(sd.listFiles().length, is(1)); + assertThat(sd.listFiles()[0].getAbsolutePath(), is(new File(sd, "keyvalue.sd").getAbsolutePath())); + + File ext = files.get(files.indexOf(new File(outApp, "external"))); + assertTrue(ext.isDirectory()); + assertThat(ext.listFiles().length, is(1)); + assertThat(ext.listFiles()[0].getAbsolutePath(), is(new File(ext, "foo").getAbsolutePath())); + + files = Arrays.asList(ext.listFiles()); + File foo = files.get(files.indexOf(new File(ext, "foo"))); + assertTrue(foo.isDirectory()); + assertThat(foo.listFiles().length, is(1)); + assertThat(foo.listFiles()[0].getAbsolutePath(), is(new File(foo, "bar").getAbsolutePath())); + + files = Arrays.asList(foo.listFiles()); + File bar = files.get(files.indexOf(new File(foo, "bar"))); + assertTrue(bar.isDirectory()); + assertThat(bar.listFiles().length, is(1)); + assertTrue(bar.listFiles()[0].isFile()); + assertThat(bar.listFiles()[0].getAbsolutePath(), is(new File(bar, "lol").getAbsolutePath())); + } + + + @Test(expected = IOException.class) + public void require_that_invalid_application_returns_error_when_unpacked() throws IOException { + File app = new File("src/test/resources/deploy/validapp/services.xml"); + CompressedApplicationInputStream.createFromCompressedStream( + new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(app)))); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigResponseFactoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigResponseFactoryTest.java new file mode 100644 index 00000000000..f127d8716aa --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigResponseFactoryTest.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import org.junit.Before; +import org.junit.Test; + +import java.io.StringReader; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.19 + */ +public class ConfigResponseFactoryTest { + private InnerCNode def; + + + @Before + public void setup() { + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + def = dParser.getTree(); + } + + @Test + public void testUncompressedFacory() { + UncompressedConfigResponseFactory responseFactory = new UncompressedConfigResponseFactory(); + ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty(), def, 3); + assertThat(response.getCompressionInfo().getCompressionType(), is(CompressionType.UNCOMPRESSED)); + assertThat(response.getGeneration(), is(3l)); + assertThat(response.getPayload().getByteLength(), is(2)); + } + + @Test + public void testLZ4CompressedFacory() { + LZ4ConfigResponseFactory responseFactory = new LZ4ConfigResponseFactory(); + ConfigResponse response = responseFactory.createResponse(ConfigPayload.empty(), def, 3); + assertThat(response.getCompressionInfo().getCompressionType(), is(CompressionType.LZ4)); + assertThat(response.getGeneration(), is(3l)); + assertThat(response.getPayload().getByteLength(), is(3)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java new file mode 100644 index 00000000000..6f5b51e7914 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.version.VersionState; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.Optional; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigServerBootstrapTest extends TestWithTenant { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testConfigServerBootstrap() throws Exception { + File versionFile = temporaryFolder.newFile(); + ConfigserverConfig.Builder config = new ConfigserverConfig.Builder(); + MockTenantRequestHandler myServer = new MockTenantRequestHandler(Metrics.createTestMetrics()); + MockRpc rpc = new MockRpc(new ConfigserverConfig(config).rpcport()); + + assertFalse(myServer.started); + assertFalse(myServer.stopped); + VersionState versionState = new VersionState(versionFile); + assertTrue(versionState.isUpgraded()); + ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tenants, rpc, (application, timeout) -> Optional.empty(), versionState); + assertFalse(versionState.isUpgraded()); + assertThat(versionState.currentVersion(), is(versionState.storedVersion())); + assertThat(IOUtils.readAll(new FileReader(versionFile)), is(versionState.currentVersion().toSerializedForm())); + waitUntilStarted(rpc, 60000); + assertTrue(rpc.started); + assertFalse(rpc.stopped); + bootstrap.deconstruct(); + assertTrue(rpc.started); + assertTrue(rpc.stopped); + } + + private void waitUntilStarted(MockRpc server, long timeout) throws InterruptedException { + long start = System.currentTimeMillis(); + while ((System.currentTimeMillis() - start) < timeout) { + if (server.started) + return; + Thread.sleep(10); + } + } + + public static class MockTenantRequestHandler extends TenantRequestHandler { + public volatile boolean started = false; + public volatile boolean stopped = false; + + public MockTenantRequestHandler(Metrics statistics) { + super(statistics, TenantName.from("testTenant"), new ArrayList<>(), new UncompressedConfigResponseFactory(), new HostRegistries()); + } + } + + public static class MockRpc extends com.yahoo.vespa.config.server.MockRpc { + public volatile boolean started = false; + public volatile boolean stopped = false; + + public MockRpc(int port) { + super(port); + } + + @Override + public void run() { + started = true; + } + + @Override + public void stop() { + stopped = true; + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java new file mode 100644 index 00000000000..0b07fd5b8e7 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerDBTest.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.io.Files; +import com.yahoo.io.IOUtils; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigServerDBTest { + private ConfigServerDB serverDB; + private File tempDir; + + @Before + public void setup() { + tempDir = Files.createTempDir(); + serverDB = ConfigServerDB.createTestConfigServerDb(tempDir.getAbsolutePath()); + } + + private ConfigServerDB createInitializer(File pluginDir) throws IOException { + File existingDef = new File(serverDB.classes(), "test.def"); + IOUtils.writeFile(existingDef, "hello", false); + return ConfigServerDB.createTestConfigServerDb(tempDir.getAbsolutePath()); + } + + @Test + public void require_that_existing_def_files_are_copied() throws IOException { + assertThat(serverDB.serverdefs().listFiles().length, is(0)); + createInitializer(Files.createTempDir()); + assertThat(serverDB.serverdefs().listFiles().length, is(1)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/DelayedConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/DelayedConfigResponseTest.java new file mode 100644 index 00000000000..7f9ac50c549 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/DelayedConfigResponseTest.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; + +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Trace; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class DelayedConfigResponseTest { + + @Test + public void testDelayedConfigResponses() { + + MockRpc rpc = new MockRpc(13337); + DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); + assertThat(responses.size(), is(0)); + JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 1000000, "bar"); + req.setDelayedResponse(true); + GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId()); + responses.delayResponse(req, context); + assertThat(responses.size(), is(0)); + + req.setDelayedResponse(false); + responses.delayResponse(req, context); + responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); + assertThat(responses.size(), is(2)); + assertTrue(req.isDelayedResponse()); + List<DelayedConfigResponses.DelayedConfigResponse> it = responses.allDelayedResponses(); + assertTrue(!it.isEmpty()); + } + + @Test + public void testDelayResponseRemove() { + GetConfigContext context = GetConfigContext.testContext(ApplicationId.defaultId()); + MockRpc rpc = new MockRpc(13337); + DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); + responses.delayResponse(createRequest("foolio", "md5", "myid", "mymd5", 3, 100000, "bar"), context); + assertThat(responses.size(), is(1)); + responses.allDelayedResponses().get(0).cancel(); + assertThat(responses.size(), is(0)); + } + + @Test + public void testDelayedConfigResponse() { + MockRpc rpc = new MockRpc(13337); + DelayedConfigResponses responses = new DelayedConfigResponses(rpc, 1, false); + assertThat(responses.size(), is(0)); + assertThat(responses.toString(), is("DelayedConfigResponses. Average Size=0")); + JRTServerConfigRequest req = createRequest("foo", "md5", "myid", "mymd5", 3, 100, "bar"); + responses.delayResponse(req, GetConfigContext.testContext(ApplicationId.defaultId())); + rpc.waitUntilSet(5000); + assertThat(rpc.latestRequest, is(req)); + } + + public JRTServerConfigRequest createRequest(String configName, String defMd5, String configId, String md5, long generation, long timeout, String namespace) { + Request request = JRTClientConfigRequestV3. + createWithParams(new ConfigKey<>(configName, configId, namespace, defMd5, null), DefContent.fromList(Collections.emptyList()), + "fromHost", md5, generation, timeout, Trace.createDummy(), CompressionType.UNCOMPRESSED, + Optional.empty()).getRequest(); + return JRTServerConfigRequestV3.createFromRequest(request); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/DeployHandlerLoggerTest.java new file mode 100644 index 00000000000..b34881bcba8 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/DeployHandlerLoggerTest.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.log.LogLevel; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class DeployHandlerLoggerTest { + @Test + public void test_verbose_logging() throws IOException { + testLogging(true, ".*time.*level\":\"DEBUG\".*message.*time.*level\":\"SPAM\".*message.*time.*level\":\"FINE\".*message.*time.*level\":\"WARNING\".*message.*"); + } + + @Test + public void test_normal_logging() throws IOException { + testLogging(false, ".*\\{\"time.*level\":\"WARNING\".*message.*"); + } + + private void testLogging(boolean verbose, String expectedPattern) throws IOException { + Slime slime = new Slime(); + Cursor array = slime.setArray(); + DeployLogger logger = new DeployHandlerLogger(array, verbose, new ApplicationId.Builder() + .tenant("testtenant").applicationName("testapp").build()); + logMessages(logger); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + new JsonFormat(true).encode(baos, slime); + assertTrue(Pattern.matches(expectedPattern, baos.toString())); + } + + private void logMessages(DeployLogger logger) { + logger.log(LogLevel.DEBUG, "foobar"); + logger.log(LogLevel.SPAM, "foobar"); + logger.log(LogLevel.FINE, "baz"); + logger.log(LogLevel.WARNING, "baz"); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/GetConfigProcessorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/GetConfigProcessorTest.java new file mode 100644 index 00000000000..cd37b4a31c7 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/GetConfigProcessorTest.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.provision.TenantName; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.ConfigKey; + +import com.yahoo.vespa.config.protocol.CompressionInfo; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Trace; +import org.junit.Test; +import static org.junit.Assert.assertFalse; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class GetConfigProcessorTest { + + @Test + public void testSentinelConfig() { + MockRpc rpc = new MockRpc(13337, false); + rpc.response = new MockConfigResponse("foo"); // should be a sentinel config, but it does not matter for this test + + // one tenant, which has host1 assigned + boolean pretentToHaveLoadedApplications = true; + TenantName testTenant = TenantName.from("test"); + rpc.onTenantCreate(testTenant, new MockTenantProvider(pretentToHaveLoadedApplications)); + rpc.hostsUpdated(testTenant, Collections.singleton("host1")); + + { // a config is returned normally + JRTServerConfigRequest req = createV3SentinelRequest("host1"); + GetConfigProcessor proc = new GetConfigProcessor(rpc, req, false); + proc.run(); + assertTrue(rpc.tryResolveConfig); + assertTrue(rpc.tryRespond); + assertThat(rpc.errorCode, is(0)); + } + + rpc.resetChecks(); + // host1 is replaced by host2 for this tenant + rpc.hostsUpdated(testTenant, Collections.singleton("host2")); + + { // this causes us to get an empty config instead of normal config resolution + JRTServerConfigRequest req = createV3SentinelRequest("host1"); + GetConfigProcessor proc = new GetConfigProcessor(rpc, req, false); + proc.run(); + assertFalse(rpc.tryResolveConfig); // <-- no normal config resolution happening + assertTrue(rpc.tryRespond); + assertThat(rpc.errorCode, is(0)); + } + } + + private static JRTServerConfigRequest createV3SentinelRequest(String fromHost) { + final ConfigKey<?> configKey = new ConfigKey<>(SentinelConfig.CONFIG_DEF_NAME, "myid", SentinelConfig.CONFIG_DEF_NAMESPACE); + return JRTServerConfigRequestV3.createFromRequest(JRTClientConfigRequestV3. + createWithParams(configKey, DefContent.fromList(Arrays.asList(SentinelConfig.CONFIG_DEF_SCHEMA)), + fromHost, "", 0, 100, Trace.createDummy(), CompressionType.UNCOMPRESSED, + Optional.empty()).getRequest()); + } + + private class MockConfigResponse implements ConfigResponse { + + private final String line; + public MockConfigResponse(String line) { + this.line = line; + } + + @Override + public Utf8Array getPayload() { + return new Utf8String(""); + } + + @Override + public List<String> getLegacyPayload() { + return Arrays.asList(line); + } + + @Override + public long getGeneration() { + return 1; + } + + @Override + public String getConfigMd5() { + return "mymd5"; + } + + @Override + public void serialize(OutputStream os, CompressionType uncompressed) throws IOException { + os.write(Utf8.toBytes(line)); + } + + @Override + public CompressionInfo getCompressionInfo() { + return CompressionInfo.uncompressed(); + } + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/HostRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/HostRegistryTest.java new file mode 100644 index 00000000000..1147d2d4c0d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/HostRegistryTest.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.junit.Test; + +import static org.hamcrest.collection.IsIterableContainingInOrder.contains; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.3 + */ +public class HostRegistryTest { + @Test + public void old_hosts_are_removed() { + HostRegistry<String> reg = new HostRegistry<>(); + assertNull(reg.getKeyForHost("foo.com")); + reg.update("fookey", Arrays.asList("foo.com", "bar.com", "baz.com")); + assertGetKey(reg, "foo.com", "fookey"); + assertGetKey(reg, "bar.com", "fookey"); + assertGetKey(reg, "baz.com", "fookey"); + assertThat(reg.getAllHosts().size(), is(3)); + reg.update("fookey", Arrays.asList("bar.com", "baz.com")); + assertNull(reg.getKeyForHost("foo.com")); + assertGetKey(reg, "bar.com", "fookey"); + assertGetKey(reg, "baz.com", "fookey"); + + assertThat(reg.getAllHosts().size(), is(2)); + assertThat(reg.getAllHosts(), contains("bar.com", "baz.com")); + reg.removeHostsForKey("fookey"); + assertThat(reg.getAllHosts().size(), is(0)); + assertNull(reg.getKeyForHost("foo.com")); + assertNull(reg.getKeyForHost("bar.com")); + } + + @Test + public void multiple_keys_are_handled() { + HostRegistry<String> reg = new HostRegistry<>(); + reg.update("fookey", Arrays.asList("foo.com", "bar.com")); + reg.update("barkey", Arrays.asList("baz.com", "quux.com")); + assertGetKey(reg, "foo.com", "fookey"); + assertGetKey(reg, "bar.com", "fookey"); + assertGetKey(reg, "baz.com", "barkey"); + assertGetKey(reg, "quux.com", "barkey"); + } + + @Test(expected = IllegalArgumentException.class) + public void keys_cannot_overlap() { + HostRegistry<String> reg = new HostRegistry<>(); + reg.update("fookey", Arrays.asList("foo.com", "bar.com")); + reg.update("barkey", Arrays.asList("bar.com", "baz.com")); + } + + @Test + public void all_hosts_are_returned() { + HostRegistry<String> reg = new HostRegistry<>(); + reg.update("fookey", Arrays.asList("foo.com", "bar.com")); + reg.update("barkey", Arrays.asList("baz.com", "quux.com")); + assertThat(reg.getAllHosts().size(), is(4)); + } + + @Test + public void ensure_that_collection_is_copied() { + HostRegistry<String> reg = new HostRegistry<>(); + List<String> hosts = new ArrayList<>(Arrays.asList("foo.com", "bar.com", "baz.com")); + reg.update("fookey", hosts); + assertThat(reg.getCurrentHosts("fookey").size(), is(3)); + hosts.remove(2); + assertThat(reg.getCurrentHosts("fookey").size(), is(3)); + } + + @Test + public void ensure_that_underlying_hosts_do_not_change() { + HostRegistry<String> reg = new HostRegistry<>(); + reg.update("fookey", new ArrayList<>(Arrays.asList("foo.com", "bar.com", "baz.com"))); + Collection<String> hosts = reg.getAllHosts(); + assertThat(hosts.size(), is(3)); + reg.update("fookey", new ArrayList<>(Arrays.asList("foo.com"))); + assertThat(hosts.size(), is(3)); + } + + private void assertGetKey(HostRegistry<String> reg, String host, String expectedKey) { + assertNotNull(reg.getKeyForHost(host)); + assertThat(reg.getKeyForHost(host), is(expectedKey)); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java new file mode 100644 index 00000000000..09576d18b32 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.http.v2.SessionActiveHandlerTest; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.*; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.model.VespaModelFactory; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class InjectedGlobalComponentRegistryTest { + + private Curator curator; + private ConfigCurator configCurator; + private Metrics metrics; + private ConfigServerDB serverDB; + private SessionPreparer sessionPreparer; + private ConfigserverConfig configserverConfig; + private RpcServer rpcServer; + private SuperModelGenerationCounter generationCounter; + private ConfigDefinitionRepo defRepo; + private PermanentApplicationPackage permanentApplicationPackage; + private HostRegistries hostRegistries; + private GlobalComponentRegistry globalComponentRegistry; + private ModelFactoryRegistry modelFactoryRegistry; + private HostProvisionerProvider hostProvisionerProvider; + private Zone zone; + + @Before + public void setupRegistry() { + curator = new MockCurator(); + configCurator = ConfigCurator.create(curator); + metrics = Metrics.createTestMetrics(); + modelFactoryRegistry = new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); + configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(Files.createTempDir().getAbsolutePath())); + serverDB = new ConfigServerDB(configserverConfig); + sessionPreparer = new SessionTest.MockSessionPreparer(); + rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), new HostRegistries()); + generationCounter = new SuperModelGenerationCounter(curator); + defRepo = new StaticConfigDefinitionRepo(); + permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); + hostRegistries = new HostRegistries(); + hostProvisionerProvider = HostProvisionerProvider.withProvisioner(new SessionActiveHandlerTest.MockProvisioner()); + zone = Zone.defaultZone(); + globalComponentRegistry = new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, serverDB, sessionPreparer, rpcServer, configserverConfig, generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone); + } + + @Test + public void testThatAllComponentsAreSetup() { + assertThat(globalComponentRegistry.getModelFactoryRegistry(), is(modelFactoryRegistry)); + assertThat(globalComponentRegistry.getServerDB(), is(serverDB)); + assertThat(globalComponentRegistry.getSessionPreparer(), is(sessionPreparer)); + assertThat(globalComponentRegistry.getMetrics(), is(metrics)); + assertThat(globalComponentRegistry.getCurator(), is(curator)); + assertThat(globalComponentRegistry.getConfigserverConfig(), is(configserverConfig)); + assertThat(globalComponentRegistry.getReloadListener().hashCode(), is(rpcServer.hashCode())); + assertThat(globalComponentRegistry.getTenantListener().hashCode(), is(rpcServer.hashCode())); + assertThat(globalComponentRegistry.getSuperModelGenerationCounter(), is(generationCounter)); + assertThat(globalComponentRegistry.getConfigDefinitionRepo(), is(defRepo)); + assertThat(globalComponentRegistry.getPermanentApplicationPackage(), is(permanentApplicationPackage)); + assertThat(globalComponentRegistry.getHostRegistries(), is(hostRegistries)); + assertThat(globalComponentRegistry.getZone(), is (zone)); + assertTrue(globalComponentRegistry.getHostProvisioner().isPresent()); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MemoryGenerationCounter.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MemoryGenerationCounter.java new file mode 100644 index 00000000000..461da73638c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MemoryGenerationCounter.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.config.GenerationCounter; + +/** + * @author lulf + * @since 5. + */ +public class MemoryGenerationCounter implements GenerationCounter { + long value; + @Override + public long increment() { + return ++value; + } + + @Override + public long get() { + return value; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java new file mode 100644 index 00000000000..1ba660dc5d1 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MiscTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import static org.junit.Assert.*; +import java.io.*; +import java.util.List; +import java.util.ArrayList; +import org.junit.Test; +import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.config.AppConfig; +import com.yahoo.config.Md5testConfig; + +/** + * Tests that does not yet have a specific home due to removed classes, obsolete features etc. + * + * @author vegardh + */ +public class MiscTestCase { + + /** + * Verifies that the md5 sum computed on the server is equal to that in the generated class. + * + * @throws java.io.IOException if an error in zk + */ + @Test + public void testGetDefMd5() throws IOException { + System.out.println("\nStarting testGetDefMd5"); + final String defDir = "src/test/resources/configdefinitions/"; + assertEquals(AppConfig.CONFIG_DEF_MD5, ConfigUtils.getDefMd5(file2lines(new File(defDir + "app.def")))); + assertEquals(Md5testConfig.CONFIG_DEF_MD5, ConfigUtils.getDefMd5(file2lines(new File(defDir + "md5test.def")))); + } + + private static List<String> file2lines(File file) throws IOException { + List<String> lines = new ArrayList<>(); + LineNumberReader in = new LineNumberReader(new InputStreamReader(new FileInputStream(file), "UTF-8")); + String line; + while ((line = in.readLine()) != null) { + lines.add(line); + } + return lines; + } + + @Test + public void testMd5StripSpaces() { + assertEquals("", ConfigUtils.stripSpaces("")); + assertEquals("foo", ConfigUtils.stripSpaces("foo")); + assertEquals(" foo", ConfigUtils.stripSpaces(" foo")); + assertEquals("bar ", ConfigUtils.stripSpaces("bar ")); + assertEquals("bar ", ConfigUtils.stripSpaces("bar ")); + assertEquals("b ar", ConfigUtils.stripSpaces("b \t ar")); + assertEquals("bar foo", ConfigUtils.stripSpaces("bar\t\tfoo")); + assertEquals("blabla string default=\"\t\"", ConfigUtils.stripSpaces("blabla string default=\"\t\"")); + assertEquals("blabla string default=\"foo\tbar\"", ConfigUtils.stripSpaces("blabla string default=\"foo\tbar\"")); + assertEquals("blabla string default=\" \t \"", ConfigUtils.stripSpaces("blabla string default=\" \t \"")); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java new file mode 100644 index 00000000000..d705203b5af --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author lulf + * @since 5.1.24 + */ +public class MockReloadHandler implements ReloadHandler { + public ApplicationSet current = null; + public ReloadListener listener = null; + public volatile ApplicationId lastRemoved = null; + + @Override + public void reloadConfig(ApplicationSet application) { + this.current = application; + } + + @Override + public void removeApplication(ApplicationId applicationId) { + lastRemoved = applicationId; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockRequestHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockRequestHandler.java new file mode 100644 index 00000000000..373545f1a8b --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockRequestHandler.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; + +import java.util.*; + +/** + * Test utility class + * @author lulf + * @since 5.25 + */ +public class MockRequestHandler implements RequestHandler, ReloadHandler, TenantHandlerProvider { + + volatile String serverStats = ""; + volatile boolean reloadResponse = false; + volatile boolean throwException = false; + public long appGeneration = 0; + private Set<ConfigKey<?>> allConfigs = new HashSet<>(); + public volatile ConfigResponse responseConfig = null; // for some v1 mocking + public Map<ApplicationId, ConfigResponse> responses = new LinkedHashMap<>(); // for v2 mocking + private final boolean pretendToHaveLoadedAnyApplication; + + public MockRequestHandler() { + this(false); + } + + public MockRequestHandler(boolean pretendToHaveLoadedAnyApplication) { + this.pretendToHaveLoadedAnyApplication = pretendToHaveLoadedAnyApplication; + } + + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + if (appId==null) { + checkThrow(); + return responseConfig; + } + return responses.get(appId); + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + return Collections.emptySet(); + } + + @Override + public void removeApplication(ApplicationId applicationId) { + } + + @Override + public void reloadConfig(ApplicationSet application) { + checkThrow(); + } + + private void checkThrow() { + if (throwException) { + throw new RuntimeException("foo"); + } + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> key, boolean recursive) { + return Collections.emptySet(); + } + + @Override + public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { + Set<String> ret = new HashSet<>(); + for (ConfigKey<?> k : allConfigs) { + ret.add(k.getConfigId()); + } + return ret; + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + return allConfigs; + } + + public void setAllConfigs(Set<ConfigKey<?>> allConfigs) { + this.allConfigs = allConfigs; + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + if (pretendToHaveLoadedAnyApplication) return true; + return responses.containsKey(appId); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + return ApplicationId.defaultId(); + } + + @Override + public RequestHandler getRequestHandler() { + return this; + } + + @Override + public ReloadHandler getReloadHandler() { + return this; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockRpc.java new file mode 100644 index 00000000000..70e4b4f6bd0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockRpc.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; +import com.yahoo.vespa.config.server.monitoring.Metrics; + +import java.util.Optional; +import java.util.concurrent.CompletionService; + +/** + * Test utility mocking an RPC server. + * + * @author lulf + * @since 5.25 + */ +public class MockRpc extends RpcServer { + + public boolean forced = false; + public RuntimeException exception = null; + public int errorCode = 0; + public ConfigResponse response = null; + + // Fields used to assert on the calls made to this from tests + public boolean tryResolveConfig = false; + public boolean tryRespond = false; + /** The last request received and responded to */ + public volatile JRTServerConfigRequest latestRequest = null; + + + public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) { + super(createConfig(port), null, Metrics.createTestMetrics(), new HostRegistries()); + if (createDefaultTenant) { + onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); + } + } + + public MockRpc(int port, boolean createDefaultTenant) { + this(port, createDefaultTenant, true); + } + + public MockRpc(int port) { + this(port, true); + } + + /** Reset fields used to assert on the calls made to this */ + public void resetChecks() { + forced = false; + tryResolveConfig = false; + tryRespond = false; + latestRequest = null; + } + + private static ConfigserverConfig createConfig(int port) { + ConfigserverConfig.Builder b = new ConfigserverConfig.Builder(); + b.rpcport(port); + return new ConfigserverConfig(b); + } + + public boolean waitUntilSet(int timeout) { + long start = System.currentTimeMillis(); + long end = start + timeout; + while (start < end) { + if (latestRequest != null) + return true; + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + return false; + } + + @Override + public Boolean addToRequestQueue(JRTServerConfigRequest request, boolean forceResponse, CompletionService<Boolean> completionService) { + latestRequest = request; + forced = forceResponse; + return true; + } + + @Override + public void respond(JRTServerConfigRequest request) { + latestRequest = request; + tryRespond = true; + errorCode = request.errorCode(); + } + + @Override + public ConfigResponse resolveConfig(JRTServerConfigRequest request, GetConfigContext context, Optional<Version> vespaVersion) { + tryResolveConfig = true; + if (exception != null) { + throw exception; + } + return response; + } + + @Override + public boolean isHostedVespa() { return true; } + + @Override + public boolean allTenantsLoaded() { return true; } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTenantListener.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTenantListener.java new file mode 100644 index 00000000000..1fd92f2acc9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTenantListener.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.TenantName; + +/** + * @author lulf + * @since 5.8 + */ +public class MockTenantListener implements TenantListener { + TenantName tenantCreatedName; + TenantHandlerProvider provider; + TenantName tenantDeletedName; + boolean tenantsLoaded; + + @Override + public void onTenantCreate(TenantName tenantName, TenantHandlerProvider provider) { + this.tenantCreatedName = tenantName; + this.provider = provider; + } + + @Override + public void onTenantDelete(TenantName tenantName) { + this.tenantDeletedName = tenantName; + } + + @Override + public void onTenantsLoaded() { + tenantsLoaded = true; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTenantProvider.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTenantProvider.java new file mode 100644 index 00000000000..4b92c91f581 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTenantProvider.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +/** + * @author lulf + * @since 5. + */ +public class MockTenantProvider implements TenantHandlerProvider { + + final MockRequestHandler requestHandler; + final MockReloadHandler reloadHandler; + + public MockTenantProvider() { + this(false); + } + + public MockTenantProvider(boolean pretendToHaveLoadedAnyApplication) { + this.requestHandler = new MockRequestHandler(pretendToHaveLoadedAnyApplication); + this.reloadHandler = new MockReloadHandler(); + } + + @Override + public RequestHandler getRequestHandler() { return requestHandler; } + + @Override + public ReloadHandler getReloadHandler() { + return reloadHandler; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java new file mode 100644 index 00000000000..0352f2c9f3e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.deploy.ModelContextImpl; + +import org.junit.Test; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + */ +public class ModelContextImplTest { + @Test + public void testModelContextTest() { + + final Rotation rotation = new Rotation("this.is.a.mock.rotation"); + final Set<Rotation> rotations = Collections.singleton(rotation); + + ModelContext context = new ModelContextImpl( + MockApplicationPackage.createEmpty(), + Optional.empty(), + Optional.empty(), + new BaseDeployLogger(), + new StaticConfigDefinitionRepo(), + new MockFileRegistry(), + Optional.empty(), + new ModelContextImpl.Properties( + ApplicationId.defaultId(), + true, + Collections.emptyList(), + false, + Zone.defaultZone(), + rotations), + Optional.empty(), + Optional.empty()); + assertTrue(context.applicationPackage() instanceof MockApplicationPackage); + assertFalse(context.hostProvisioner().isPresent()); + assertFalse(context.permanentApplicationPackage().isPresent()); + assertFalse(context.previousModel().isPresent()); + assertTrue(context.getFileRegistry() instanceof MockFileRegistry); + assertTrue(context.configDefinitionRepo() instanceof StaticConfigDefinitionRepo); + assertThat(context.properties().applicationId(), is(ApplicationId.defaultId())); + assertTrue(context.properties().configServerSpecs().isEmpty()); + assertTrue(context.properties().multitenant()); + assertTrue(context.properties().zone() instanceof Zone); + assertFalse(context.properties().hostedVespa()); + assertThat(context.properties().rotations(), equalTo(rotations)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java new file mode 100644 index 00000000000..3d0c50b9bcb --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + */ +public class ModelFactoryRegistryTest { + @Test(expected = IllegalArgumentException.class) + public void testThatOneFactoryIsRequired() { + new ModelFactoryRegistry(new ComponentRegistry<>()); + } + + @Test + public void testThatLatestVersionIsSelected() { + Version versionA = Version.fromIntValues(5, 38, 4); + Version versionB = Version.fromIntValues(5, 58, 1); + Version versionC = Version.fromIntValues(5, 48, 44); + Version versionD = Version.fromIntValues(5, 18, 44); + TestFactory a = new TestFactory(versionA); + TestFactory b = new TestFactory(versionB); + TestFactory c = new TestFactory(versionC); + TestFactory d = new TestFactory(versionD); + + for (int i = 0; i < 100; i++) { + List<ModelFactory> randomOrder = Arrays.asList(a, b, c, d); + Collections.shuffle(randomOrder); + ModelFactoryRegistry registry = new ModelFactoryRegistry(randomOrder); + assertThat(registry.getFactory(versionA), is(a)); + assertThat(registry.getFactory(versionB), is(b)); + assertThat(registry.getFactory(versionC), is(c)); + assertThat(registry.getFactory(versionD), is(d)); + } + } + + @Test + public void testThatAllFactoriesAreReturned() { + TestFactory a = new TestFactory(Version.fromIntValues(5, 38, 4)); + TestFactory b = new TestFactory(Version.fromIntValues(5, 58, 1)); + TestFactory c = new TestFactory(Version.fromIntValues(5, 48, 44)); + TestFactory d = new TestFactory(Version.fromIntValues(5, 18, 44)); + ModelFactoryRegistry registry = new ModelFactoryRegistry(Arrays.asList(a, b, c, d)); + assertThat(registry.getFactories().size(), is(4)); + assertTrue(registry.getFactories().contains(a)); + assertTrue(registry.getFactories().contains(b)); + assertTrue(registry.getFactories().contains(c)); + assertTrue(registry.getFactories().contains(d)); + } + + @Test(expected = UnknownVespaVersionException.class) + public void testThatUnknownVersionGivesError() { + ModelFactoryRegistry registry = new ModelFactoryRegistry(Arrays.asList(new TestFactory(Version.fromIntValues(1, 2, 3)))); + registry.getFactory(Version.fromIntValues(3, 2, 1)); + } + + private static class TestFactory implements ModelFactory { + private final Version version; + + public TestFactory(Version version) { + this.version = version; + } + + @Override + public Version getVersion() { + return version; + } + + @Override + public Model createModel(ModelContext modelContext) { + return null; + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + return null; + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java new file mode 100644 index 00000000000..2615622fce0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; + +import java.io.IOException; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; + +/** + * @author bratseth + */ +public class ModelStub implements Model { + + @Override + public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition targetDef, ConfigPayload override) throws IOException { + return null; + } + + @Override + public ConfigPayload getConfig(ConfigKey<?> configKey, InnerCNode targetDef, ConfigPayload override) throws IOException { + return null; + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced() { + return null; + } + + @Override + public Collection<HostInfo> getHosts() { + return null; + } + + @Override + public Set<String> allConfigIds() { + return null; + } + + @Override + public void distributeFiles(FileDistribution fileDistribution) { + + } + + @Override + public Optional<ProvisionInfo> getProvisionInfo() { + return null; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java b/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java new file mode 100644 index 00000000000..0c76a907d82 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/PortRangeAllocator.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.collect.ContiguousSet; +import com.google.common.collect.DiscreteDomain; +import com.google.common.collect.Range; + +import java.util.HashSet; +import java.util.Set; +import java.util.Stack; + +/** + * Allocates port ranges for all configserver tests. + * + * @author lulf + * @since 5.1.26 + */ +public class PortRangeAllocator { + private final static PortRange portRange = new PortRange(); + + // Get the next port from a pre-allocated range + public static int findAvailablePort() throws InterruptedException { + return portRange.next(); + } + + public static void releasePort(int port) { + portRange.release(port); + } + + private static class PortRange { + private final Set<Integer> takenPorts = new HashSet<>(); + private final Stack<Integer> freePorts = new Stack<>(); + private static final int first = 18651; + private static final int last = 18899; // see: factory/doc/port-ranges + + public PortRange() { + freePorts.addAll(ContiguousSet.create(Range.closed(first, last), DiscreteDomain.integers())); + } + + synchronized int next() throws InterruptedException { + if (freePorts.isEmpty()) { + wait(600_000); + if (freePorts.isEmpty()) { + throw new RuntimeException("no more ports in range " + first + "-" + last); + } + } + int port = freePorts.pop(); + takenPorts.add(port); + return port; + } + + synchronized void release(int port) { + if (port < first || port > last) { + throw new RuntimeException("trying to release port outside valid range " + port); + } + if (!takenPorts.contains(port)) { + throw new RuntimeException("trying to release port never acquired " + port); + } + takenPorts.remove(port); + freePorts.push(port); + notify(); + } + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/RpcServerTest.java new file mode 100644 index 00000000000..d205bfe05e6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/RpcServerTest.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.base.Joiner; +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.protocol.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.util.ConfigUtils; + +import com.yahoo.vespa.model.VespaModel; +import org.junit.*; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class RpcServerTest extends TestWithRpc { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testRpcServer() throws IOException, SAXException, InterruptedException { + testPrintStatistics(); + testGetConfig(); + testEnabled(); + testEmptyConfigHostedVespa(); + } + + + private void testEmptyConfigHostedVespa() throws InterruptedException { + rpcServer.onTenantDelete(TenantName.defaultName()); + rpcServer.onTenantsLoaded(); + JRTClientConfigRequest clientReq = createSimpleRequest(); + performRequest(clientReq.getRequest()); + assertFalse(clientReq.validateResponse()); + assertThat(clientReq.errorCode(), is(ErrorCode.APPLICATION_NOT_LOADED)); + stopRpc(); + createAndStartRpcServer(true); + rpcServer.onTenantsLoaded(); + clientReq = createSimpleRequest(); + performRequest(clientReq.getRequest()); + assertTrue(clientReq.validateResponse()); + } + + private JRTClientConfigRequest createSimpleRequest() { + ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, ""); + JRTClientConfigRequest clientReq = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, SimpletypesConfig.CONFIG_DEF_MD5), 120_000, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()); + assertTrue(clientReq.validateParameters()); + return clientReq; + } + + + private void testEnabled() throws IOException, SAXException { + generationCounter.increment(); + Application app = new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), 2l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), ApplicationId.defaultId()); + ApplicationSet appSet = ApplicationSet.fromSingle(app); + rpcServer.configReloaded(TenantName.defaultName(), appSet); + ConfigKey<?> key = new ConfigKey<>(LbServicesConfig.class, "*"); + JRTClientConfigRequest clientReq = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, LbServicesConfig.CONFIG_DEF_MD5), 120_000, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()); + assertTrue(clientReq.validateParameters()); + performRequest(clientReq.getRequest()); + assertFalse(clientReq.validateResponse()); + assertThat(clientReq.errorCode(), is(ErrorCode.APPLICATION_NOT_LOADED)); + + rpcServer.onTenantsLoaded(); + clientReq = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, LbServicesConfig.CONFIG_DEF_MD5), 120_000, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()); + assertTrue(clientReq.validateParameters()); + performRequest(clientReq.getRequest()); + boolean validResponse = clientReq.validateResponse(); + assertTrue(clientReq.errorMessage(), validResponse); + assertThat(clientReq.errorCode(), is(0)); + } + + public void testGetConfig() { + tenantProvider.requestHandler.throwException = false; + ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, "brim"); + tenantProvider.requestHandler.responses.put(ApplicationId.defaultId(), createResponse()); + JRTClientConfigRequest req = JRTClientConfigRequestV3.createFromRaw(new RawConfig(key, SimpletypesConfig.CONFIG_DEF_MD5), 120_000, Trace.createDummy(), CompressionType.UNCOMPRESSED, Optional.empty()); + assertTrue(req.validateParameters()); + performRequest(req.getRequest()); + assertThat(req.errorCode(), is(0)); + assertTrue(req.validateResponse()); + ConfigPayload payload = ConfigPayload.fromUtf8Array(req.getNewPayload().getData()); + assertNotNull(payload); + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + new ConfigPayloadApplier<>(builder).applyPayload(payload); + SimpletypesConfig config = new SimpletypesConfig(builder); + assertThat(config.intval(), is(123)); + } + + public ConfigResponse createResponse() { + SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder(); + builder.intval(123); + SimpletypesConfig responseConfig = new SimpletypesConfig(builder); + ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig); + InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA))).getTree(); + return SlimeConfigResponse.fromConfigPayload(responsePayload, targetDef, 3l, ConfigUtils.getMd5(responsePayload)); + } + + public void testPrintStatistics() { + Request req = new Request("printStatistics"); + rpcServer.printStatistics(req); + assertThat(req.returnValues().get(0).asString(), is("Delayed responses queue size: 0")); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java new file mode 100644 index 00000000000..52179b2cf48 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.config.ConfigCacheKey; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.SlimeConfigResponse; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class ServerCacheTest { + private ServerCache cache; + + private static String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1"; + private static String defMd5_2 = "a2f8edfc965802bf6d44826f9da7e2b0"; + private static String configMd5 = "mymd5"; + private static String configMd5_2 = "mymd5_2"; + private static ConfigDefinition payload = new ConfigDefinition("mypayload", new String[0]); + private static ConfigDefinition payload_2 = new ConfigDefinition("otherpayload", new String[0]); + + private static ConfigDefinitionKey fooBarDefKey = new ConfigDefinitionKey("foo", "bar"); + private static ConfigDefinitionKey fooBazDefKey = new ConfigDefinitionKey("foo", "baz"); + private static ConfigDefinitionKey fooBimDefKey = new ConfigDefinitionKey("foo", "bim"); + + private static ConfigKey<?> fooConfigKey = new ConfigKey<>("foo", "id", "bar"); + private static ConfigKey<?> bazConfigKey = new ConfigKey<>("foo", "id2", "bar"); + + ConfigCacheKey fooBarCacheKey = new ConfigCacheKey(fooConfigKey, defMd5); + ConfigCacheKey bazQuuxCacheKey = new ConfigCacheKey(bazConfigKey, defMd5); + ConfigCacheKey fooBarCacheKeyDifferentMd5 = new ConfigCacheKey(fooConfigKey, defMd5_2); + + @Before + public void setup() { + cache = new ServerCache(); + + cache.addDef(fooBarDefKey, payload); + cache.addDef(fooBazDefKey, new com.yahoo.vespa.config.buildergen.ConfigDefinition("baz", new String[0])); + + cache.addDef(fooBimDefKey, new ConfigDefinition("mynode", new String[0])); + + cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload.getCNode(), 2, configMd5), configMd5); + cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload.getCNode(), 2, configMd5), configMd5); + cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), payload_2.getCNode(), 2, configMd5_2), configMd5_2); + } + + @Test + public void testThatCacheWorks() { + assertNotNull(cache.getDef(fooBazDefKey)); + assertThat(cache.getDef(fooBarDefKey), is(payload)); + assertThat(cache.getDef(fooBimDefKey).getCNode().getName(), is("mynode")); + ConfigResponse raw = cache.get(fooBarCacheKey); + assertThat(raw.getConfigMd5(), is(configMd5)); + } + + @Test + public void testThatCacheWorksWithSameKeyDifferentMd5() { + assertThat(cache.getDef(fooBarDefKey), is(payload)); + ConfigResponse raw = cache.get(fooBarCacheKey); + assertThat(raw.getConfigMd5(), is(configMd5)); + raw = cache.get(fooBarCacheKeyDifferentMd5); + assertThat(raw.getConfigMd5(), is(configMd5_2)); + } + + @Test + public void testThatCacheWorksWithDifferentKeySameMd5() { + assertTrue(cache.get(fooBarCacheKey) == cache.get(bazQuuxCacheKey)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java new file mode 100644 index 00000000000..a69cf547db0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModel; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.Optional; + +import com.yahoo.cloud.config.ElkConfig; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.9 + */ +public class SuperModelControllerTest { + + private static final File testApp = new File("src/test/resources/deploy/app"); + private SuperModelGenerationCounter counter; + private SuperModelController controller; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + counter = new SuperModelGenerationCounter(new MockCurator()); + controller = new SuperModelController(counter, + new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder()), + new ElkConfig(new ElkConfig.Builder())); + } + + @Test + public void test_super_model_reload() throws IOException, SAXException { + TenantName tenantA = TenantName.from("a"); + assertNotNull(controller.getHandler()); + long gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + assertNotNull(controller.getHandler()); + assertThat(controller.getHandler().getGeneration(), is(gen)); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 4l, 2)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + // Test that a new app is used when there already exist an application with the same id + ApplicationId appId = new ApplicationId.Builder().tenant(tenantA).applicationName("foo").build(); + assertThat(((TestApplication) controller.getHandler().getSuperModel().getCurrentModels().get(tenantA).get(appId)).version, is(2l)); + gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "bar", 2l, 3)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + } + + @Test + public void test_super_model_remove() throws IOException, SAXException { + TenantName tenantA = TenantName.from("a"); + TenantName tenantB = TenantName.from("b"); + long gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + controller.reloadConfig(tenantA, createApp(tenantA, "bar", 30l, 2)); + controller.reloadConfig(tenantB, createApp(tenantB, "baz", 9l, 3)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getSuperModel().getCurrentModels().size(), is(2)); + assertThat(controller.getHandler().getSuperModel().getCurrentModels().get(TenantName.from("a")).size(), is(2)); + controller.removeApplication( + new ApplicationId.Builder().tenant("a").applicationName("unknown").build()); + assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getSuperModel().getCurrentModels().size(), is(2)); + assertThat(controller.getHandler().getSuperModel().getCurrentModels().get(TenantName.from("a")).size(), is(2)); + gen = counter.increment(); + controller.removeApplication( + new ApplicationId.Builder().tenant("a").applicationName("bar").build()); + assertThat(controller.getHandler().getSuperModel().getCurrentModels().size(), is(2)); + assertThat(controller.getHandler().getSuperModel().getCurrentModels().get(TenantName.from("a")).size(), is(1)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + } + + @Test + public void test_super_model_master_generation() throws IOException, SAXException { + TenantName tenantA = TenantName.from("a"); + long masterGen = 10; + controller = new SuperModelController(counter, + new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen)), + new ElkConfig(new ElkConfig.Builder())); + + long gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + assertThat(controller.getHandler().getGeneration(), is(masterGen + gen)); + } + + @Test + public void test_super_model_has_application_when_enabled() { + assertFalse(controller.hasApplication(ApplicationId.global(), Optional.empty())); + controller.enable(); + assertTrue(controller.hasApplication(ApplicationId.global(), Optional.empty())); + } + + private ApplicationSet createApp(TenantName tenant, String application, long generation, long version) throws IOException, SAXException { + return ApplicationSet.fromSingle( + new TestApplication( + new VespaModel(FilesApplicationPackage.fromFile(testApp)), + new ServerCache(), + generation, + new ApplicationId.Builder().tenant(tenant).applicationName(application).build(), + version)); + } + + private static class TestApplication extends Application { + private long version = 0; + + public TestApplication(VespaModel vespaModel, ServerCache cache, long appGeneration, ApplicationId app, long version) { + super(vespaModel, cache, appGeneration, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app); + this.version = version; + } + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java new file mode 100644 index 00000000000..93a48094fac --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -0,0 +1,203 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.ElkConfig; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.*; +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.vespa.config.protocol.VespaVersion; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.model.SuperModel; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.model.VespaModel; + +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +import com.yahoo.cloud.config.ElkConfig.Logstash; + +import com.yahoo.vespa.config.server.model.ElkProducer; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.9 + */ +public class SuperModelRequestHandlerTest { + + private SuperModelRequestHandler handler; + + @Before + public void setupHandler() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); + models.put(TenantName.from("a"), new LinkedHashMap<>()); + File testApp = new File("src/test/resources/deploy/app"); + ApplicationId app = ApplicationId.from(TenantName.from("a"), + ApplicationName.from("foo"), InstanceName.defaultName()); + models.get(app.tenant()).put(app, new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app)); + handler = new SuperModelRequestHandler(new SuperModel(models, new ElkConfig(new ElkConfig.Builder()), Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + } + + @Test + public void test_super_model_resolve_elk() { + ConfigResponse response = handler.resolveConfig(new GetConfigRequest() { + @Override + public ConfigKey<?> getConfigKey() { + return new ConfigKey<>(ElkConfig.class, "dontcare"); + } + + @Override + public DefContent getDefContent() { + return DefContent.fromClass(ElkConfig.class); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return Optional.empty(); + } + + @Override + public boolean noCache() { + return false; + } + }); + assertThat(response.getGeneration(), is(2l)); + } + + @Test + public void test_lb_config_simple() { + LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); + handler.getSuperModel().getConfig(lb); + LbServicesConfig lbc = new LbServicesConfig(lb); + assertThat(lbc.tenants().size(), is(1)); + assertThat(lbc.tenants("a").applications().size(), is(1)); + Applications app = lbc.tenants("a").applications("foo:prod:default:default"); + assertTrue(app.hosts().size() > 0); + } + + + @Test(expected = UnknownConfigDefinitionException.class) + public void test_unknown_config_definition() { + String md5 = "asdfasf"; + Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", md5, null), DefContent.fromList(Collections.emptyList()), + "fromHost", md5, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED, + Optional.empty()) + .getRequest(); + JRTServerConfigRequestV3 v3Request = JRTServerConfigRequestV3.createFromRequest(request); + handler.resolveConfig(v3Request); + } + + @Test + public void test_lb_config_multiple_apps() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); + models.put(TenantName.from("t1"), new LinkedHashMap<>()); + models.put(TenantName.from("t2"), new LinkedHashMap<>()); + File testApp1 = new File("src/test/resources/deploy/app"); + File testApp2 = new File("src/test/resources/deploy/advancedapp"); + File testApp3 = new File("src/test/resources/deploy/advancedapp"); + // TODO must fix equals, hashCode on Tenant + Version vespaVersion = Version.fromIntValues(1, 2, 3); + models.get(TenantName.from("t1")).put(applicationId("mysimpleapp"), + new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp1)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("mysimpleapp"))); + models.get(TenantName.from("t1")).put(applicationId("myadvancedapp"), + new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp2)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("myadvancedapp"))); + models.get(TenantName.from("t2")).put(applicationId("minetooadvancedapp"), + new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp3)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("minetooadvancedapp"))); + + SuperModelRequestHandler han = new SuperModelRequestHandler(new SuperModel(models, new ElkConfig(new ElkConfig.Builder()), Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); + han.getSuperModel().getConfig(lb); + LbServicesConfig lbc = new LbServicesConfig(lb); + assertThat(lbc.tenants().size(), is(2)); + assertThat(lbc.tenants("t1").applications().size(), is(2)); + assertThat(lbc.tenants("t2").applications().size(), is(1)); + assertThat(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default").hosts().size(), is(1)); + assertQrServer(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default")); + } + + private ApplicationId applicationId(String applicationName) { + return ApplicationId.from(TenantName.defaultName(), + ApplicationName.from(applicationName), InstanceName.defaultName()); + } + + private void assertQrServer(Applications app) { + String host = app.hosts().keySet().iterator().next(); + Applications.Hosts hosts = app.hosts(host); + assertThat(hosts.hostname(), is(host)); + for (Map.Entry<String, Applications.Hosts.Services> e : app.hosts(host).services().entrySet()) { + System.out.println(e); + if ("qrserver".equals(e.getKey())) { + Applications.Hosts.Services s = e.getValue(); + assertThat(s.type(), is("qrserver")); + assertThat(s.ports().size(), is(4)); + assertThat(s.index(), is(0)); + return; + } + } + org.junit.Assert.fail("No qrserver service in config"); + } + + @Test + public void testElkConfig() { + ElkConfig ec = new ElkConfig(new ElkConfig.Builder().elasticsearch(new ElkConfig.Elasticsearch.Builder().host("es1").port(99)). + logstash(new ElkConfig.Logstash.Builder(). + config_file("/cfgfile"). + source_field("srcfield"). + spool_size(345). + network(new Logstash.Network.Builder(). + servers(new Logstash.Network.Servers.Builder(). + host("ls1"). + port(999)). + servers(new Logstash.Network.Servers.Builder(). + host("ls2"). + port(998)). + timeout(78)). + files(new ElkConfig.Logstash.Files.Builder(). + paths("path1"). + paths("path2"). + fields("field1", "f1val"). + fields("field2", "f2val")))); + ElkProducer ep = new ElkProducer(ec); + ElkConfig.Builder newBuilder = new ElkConfig.Builder(); + ep.getConfig(newBuilder); + ElkConfig elkConfig = new ElkConfig(newBuilder); + assertThat(elkConfig.elasticsearch(0).host(), is("es1")); + assertThat(elkConfig.elasticsearch(0).port(), is(99)); + assertThat(elkConfig.logstash().network().servers(0).host(), is("ls1")); + assertThat(elkConfig.logstash().network().servers(0).port(), is(999)); + assertThat(elkConfig.logstash().network().servers(1).host(), is("ls2")); + assertThat(elkConfig.logstash().network().servers(1).port(), is(998)); + assertThat(elkConfig.logstash().network().timeout(), is(78)); + assertThat(elkConfig.logstash().config_file(), is("/cfgfile")); + assertThat(elkConfig.logstash().source_field(), is("srcfield")); + assertThat(elkConfig.logstash().spool_size(), is(345)); + assertThat(elkConfig.logstash().files().size(), is(1)); + assertThat(elkConfig.logstash().files(0).paths(0), is("path1")); + assertThat(elkConfig.logstash().files(0).paths(1), is("path2")); + assertThat(elkConfig.logstash().files(0).fields("field1"), is("f1val")); + assertThat(elkConfig.logstash().files(0).fields("field2"), is("f2val")); + } + } + + + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TenantRequestHandlerTest.java new file mode 100644 index 00000000000..94e08fdf1b3 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TenantRequestHandlerTest.java @@ -0,0 +1,300 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.VespaVersion; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.VespaModelFactory; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class TenantRequestHandlerTest extends TestWithCurator { + + private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).getVersion(); + private TenantRequestHandler server; + private MockReloadListener listener = new MockReloadListener(); + private File app1 = new File("src/test/apps/cs1"); + private File app2 = new File("src/test/apps/cs2"); + private TenantName tenant = TenantName.from("mytenant"); + private TestComponentRegistry componentRegistry; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setUp() throws IOException, SAXException { + feedApp(app1, 1); + Metrics sh = Metrics.createTestMetrics(); + List<ReloadListener> listeners = new ArrayList<>(); + listeners.add(listener); + server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), new HostRegistries()); + componentRegistry = new TestComponentRegistry(curator, configCurator, createRegistry()); + } + + private void feedApp(File appDir, long sessionId) throws IOException { + feedApp(appDir, sessionId, new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build()); + } + + private void feedApp(File appDir, long sessionId, ApplicationId appId) throws IOException { + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, new PathProvider(Path.createRoot()).getSessionDir(sessionId), new TestConfigDefinitionRepo(), ""); + zkc.writeApplicationId(appId); + File app = tempFolder.newFolder(); + IOUtils.copyDirectory(appDir, app); + ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); + deployer.deploy(FilesApplicationPackage.fromFile(appDir), Collections.singletonMap(vespaVersion, new MockFileRegistry()), Collections.emptyMap()); + } + + private ApplicationSet reloadConfig(long id) { + return reloadConfig(id, "default"); + } + + private ApplicationSet reloadConfig(long id, String application) { + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, new PathProvider(Path.createRoot()).getSessionDir(id), new TestConfigDefinitionRepo(), ""); + zkc.writeApplicationId(new ApplicationId.Builder().tenant(tenant).applicationName(application).build()); + RemoteSession session = new RemoteSession(tenant, id, componentRegistry, zkc); + return session.ensureApplicationLoaded(); + } + + private ModelFactoryRegistry createRegistry() { + return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), + new TestModelFactory(Version.fromIntValues(3, 2, 1)))); + } + + public <T extends ConfigInstance> T resolve(Class<T> clazz, TenantRequestHandler tenantRequestHandler, String configId) { + return resolve(clazz, tenantRequestHandler, new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(), vespaVersion, configId); + } + + public <T extends ConfigInstance> T resolve(final Class<T> clazz, TenantRequestHandler tenantRequestHandler, ApplicationId appId, Version vespaVersion, final String configId) { + ConfigResponse response = tenantRequestHandler.resolveConfig(appId, new GetConfigRequest() { + @Override + public ConfigKey<T> getConfigKey() { + return new ConfigKey<T>(clazz, configId); + } + + @Override + public DefContent getDefContent() { + return DefContent.fromClass(clazz); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return Optional.of(VespaVersion.fromString(vespaVersion.toSerializedForm())); + } + + @Override + public boolean noCache() { + return false; + } + }, Optional.empty()); + return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); + } + + @Test + public void testReloadConfig() throws IOException, SAXException { + ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); + server.reloadConfig(reloadConfig(1)); + assertThat(listener.reloaded.get(), is(1)); + // Using only payload list for this simple test + SimpletypesConfig config = resolve(SimpletypesConfig.class, server, ""); + assertThat(config.intval(), is(1337)); + assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1l)); + + server.reloadConfig(reloadConfig(1l)); + config = resolve(SimpletypesConfig.class, server, ""); + assertThat(config.intval(), is(1337)); + assertThat(listener.reloaded.get(), is(2)); + assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1l)); + assertThat(listener.tenantHosts.size(), is(1)); + assertThat(server.resolveApplicationId("mytesthost"), is(applicationId)); + + listener.reloaded.set(0); + feedApp(app2, 2); + server.reloadConfig(reloadConfig(2l)); + config = resolve(SimpletypesConfig.class, server, ""); + assertThat(config.intval(), is(1330)); + assertThat(listener.reloaded.get(), is(1)); + assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(2l)); + } + + @Test + public void testRemoveApplication() { + server.reloadConfig(reloadConfig(1)); + assertThat(listener.removed.get(), is(0)); + server.removeApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build()); + assertThat(listener.removed.get(), is(1)); + } + + @Test + public void testResolveForAppId() { + long id = 1l; + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, configCurator, new PathProvider(Path.createRoot()).getSessionDir(id), new TestConfigDefinitionRepo(), ""); + ApplicationId appId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("myapp").instanceName("myinst").build(); + zkc.writeApplicationId(appId); + RemoteSession session = new RemoteSession(appId.tenant(), id, componentRegistry, zkc); + server.reloadConfig(session.ensureApplicationLoaded()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); + assertThat(config.intval(), is(1337)); + } + + @Test + public void testResolveMultipleApps() throws IOException, SAXException { + ApplicationId appId1 = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("myapp1").instanceName("myinst1").build(); + ApplicationId appId2 = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("myapp2").instanceName("myinst2").build(); + feedAndReloadApp(app1, 1, appId1); + SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId1, vespaVersion, ""); + assertThat(config.intval(), is(1337)); + + feedAndReloadApp(app2, 2, appId2); + config = resolve(SimpletypesConfig.class, server, appId2, vespaVersion, ""); + assertThat(config.intval(), is(1330)); + } + + @Test + public void testResolveMultipleVersions() throws IOException { + ApplicationId appId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("myapp1").instanceName("myinst1").build(); + feedAndReloadApp(app1, 1, appId); + SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); + assertThat(config.intval(), is(1337)); + config = resolve(SimpletypesConfig.class, server, appId, Version.fromIntValues(3, 2, 1), ""); + assertThat(config.intval(), is(1337)); + } + + private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { + feedApp(appDir, sessionId, appId); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, new PathProvider(Path.createRoot()).getSessionDir(sessionId)); + zkc.writeApplicationId(appId); + RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); + server.reloadConfig(session.ensureApplicationLoaded()); + } + + public static class MockReloadListener implements ReloadListener { + public AtomicInteger reloaded = new AtomicInteger(0); + public AtomicInteger removed = new AtomicInteger(0); + public Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); + @Override + public void configReloaded(TenantName tenant, ApplicationSet application) { + reloaded.incrementAndGet(); + } + + @Override + public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { + tenantHosts.put(tenant.value(), newHosts); + } + + @Override + public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { + } + + @Override + public void applicationRemoved(ApplicationId applicationId) { + removed.incrementAndGet(); + } + } + + @Test + public void testHasApplication() throws IOException, SAXException { + assertdefaultAppNotFound(); + server.reloadConfig(reloadConfig(1l)); + assertTrue(server.hasApplication(new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(), Optional.of(vespaVersion))); + } + + private void assertdefaultAppNotFound() { + assertFalse(server.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); + } + + @Test + public void testMultipleApplicationsReload() { + assertdefaultAppNotFound(); + server.reloadConfig(reloadConfig(1l, "foo")); + assertdefaultAppNotFound(); + assertTrue(server.hasApplication(new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(), + Optional.of(vespaVersion))); + assertThat(server.resolveApplicationId("doesnotexist"), is(ApplicationId.defaultId())); + assertThat(server.resolveApplicationId("mytesthost"), is(new ApplicationId.Builder() + .tenant(tenant) + .applicationName("foo").build())); // Host set in application package. + } + + @Test + public void testListConfigs() throws IOException, SAXException { + assertdefaultAppNotFound(); + /*assertTrue(server.allConfigIds(ApplicationId.defaultId()).isEmpty()); + assertTrue(server.allConfigsProduced(ApplicationId.defaultId()).isEmpty()); + assertTrue(server.listConfigs(ApplicationId.defaultId(), false).isEmpty()); + assertTrue(server.listConfigs(ApplicationId.defaultId(), true).isEmpty());*/ + + VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); + server.reloadConfig(ApplicationSet.fromSingle(new Application(model, new ServerCache(), 1, vespaVersion, MetricUpdater.createTestUpdater(), ApplicationId.defaultId()))); + Set<ConfigKey<?>> configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); + assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); + //for (ConfigKey<?> ck : configNames) { + // assertTrue(!"".equals(ck.getConfigId())); + //} + + configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); + System.out.println(configNames); + assertTrue(configNames.contains(new ConfigKey<>("feeder", "jdisc", "vespaclient.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "jdisc", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "jdisc", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "jdisc", "container.jdisc.config"))); + assertTrue(configNames.contains(new ConfigKey<>("specific", "jdisc", "project"))); + } + + @Test + public void testAppendIdsInNonRecursiveListing() { + assertEquals(server.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); + assertEquals(server.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); + assertEquals(server.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); + assertEquals(server.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TenantTest.java new file mode 100644 index 00000000000..c4eb8786917 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TenantTest.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.testing.EqualsTester; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.http.v2.TestTenantBuilder; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.3 + */ +public class TenantTest extends TestWithCurator { + + private Tenant t1; + private Tenant t2; + private Tenant t3; + private Tenant t4; + + @Before + public void setupTenant() throws Exception { + t1 = createTenant("foo"); + t2 = createTenant("foo"); + t3 = createTenant("bar"); + t4 = createTenant("baz"); + } + + private Tenant createTenant(String name) throws Exception { + return new TestTenantBuilder().createTenant(TenantName.from(name)).build(); + } + + @Test + public void equals() { + new EqualsTester() + .addEqualityGroup(t1, t2) + .addEqualityGroup(t3) + .addEqualityGroup(t4) + .testEquals(); + } + + @Test + public void hashcode() { + assertThat(t1.hashCode(), is(t2.hashCode())); + assertThat(t1.hashCode(), is(not(t3.hashCode()))); + assertThat(t1.hashCode(), is(not(t4.hashCode()))); + } + + @Test + public void close() { + MemoryApplicationRepo repo = (MemoryApplicationRepo) t1.getApplicationRepo(); + assertTrue(repo.isOpen()); + t1.close(); + assertFalse(repo.isOpen()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TenantsTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TenantsTestCase.java new file mode 100644 index 00000000000..9f71393115f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TenantsTestCase.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.deploy.MockDeployer; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.model.VespaModel; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TenantsTestCase extends TestWithCurator { + private Tenants tenants; + TestComponentRegistry globalComponentRegistry; + private TenantRequestHandlerTest.MockReloadListener listener; + private MockTenantListener tenantListener; + private final TenantName tenant1 = TenantName.from("tenant1"); + private final TenantName tenant2 = TenantName.from("tenant2"); + private final TenantName tenant3 = TenantName.from("tenant3"); + + @Before + public void setupSessions() throws Exception { + globalComponentRegistry = new TestComponentRegistry(curator); + listener = globalComponentRegistry.reloadListener; + tenantListener = globalComponentRegistry.tenantListener; + tenantListener.tenantsLoaded = false; + tenants = new Tenants(globalComponentRegistry, Metrics.createTestMetrics()); + assertTrue(tenantListener.tenantsLoaded); + tenants.createTenant(tenant1); + tenants.createTenant(tenant2); + } + + @After + public void closeSessions() throws IOException { + tenants.close(); + } + + @Test + public void testStartUp() { + assertEquals(tenants.tenantsCopy().get(tenant1).getName(), tenant1); + assertEquals(tenants.tenantsCopy().get(tenant2).getName(), tenant2); + } + + @Test + public void testListenersAdded() throws IOException, SAXException { + tenants.tenantsCopy().get(tenant1).getReloadHandler().reloadConfig(ApplicationSet.fromSingle(new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), ApplicationId.defaultId()))); + assertThat(listener.reloaded.get(), is(1)); + } + + private List<String> readZKChildren(String path) throws Exception { + return curator.framework().getChildren().forPath(path); + } + + @Test + public void testTenantListenersNotified() throws Exception { + tenants.createTenant(tenant3); + assertThat("tenant3 not the last created tenant. Tenants: " + tenants.tenantsCopy().keySet() + ", /config/v2/tenants: " + readZKChildren("/config/v2/tenants"), tenantListener.tenantCreatedName, is(tenant3)); + tenants.deleteTenant(tenant2); + assertFalse(tenants.tenantsCopy().containsKey(tenant2)); + assertThat(tenantListener.tenantDeletedName, is(tenant2)); + } + + @Test + public void testAddTenant() throws Exception { + Map<TenantName, Tenant> tenantsCopy = tenants.tenantsCopy(); + assertEquals(tenantsCopy.get(tenant1).getName(), tenant1); + assertEquals(tenantsCopy.get(tenant2).getName(), tenant2); + tenants.createTenant(tenant3); + tenantsCopy = tenants.tenantsCopy(); + assertEquals(tenantsCopy.get(tenant1).getName(), tenant1); + assertEquals(tenantsCopy.get(tenant2).getName(), tenant2); + assertEquals(tenantsCopy.get(tenant3).getName(), tenant3); + } + + @Test + public void testPutAdd() throws Exception { + tenants.createTenant(tenant3); + assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenants.tenantZkPath(tenant3))); + } + + @Test + public void testRemove() throws Exception { + assertNotNull(globalComponentRegistry.getCurator().framework().checkExists().forPath(tenants.tenantZkPath(tenant1))); + tenants.deleteTenant(tenant1); + assertFalse(tenants.tenantsCopy().containsKey(tenant1)); + } + + @Test + public void testTenantsChanged() throws Exception { + tenants.close(); // close the Tenants instance created in setupSession, we do not want to use one with a PatchChildrenCache listener + tenants = new Tenants(globalComponentRegistry, Metrics.createTestMetrics(), new ArrayList<>()); + Set<TenantName> newTenants = new LinkedHashSet<>(); + TenantName defaultTenant = TenantName.defaultName(); + newTenants.add(tenant2); + newTenants.add(defaultTenant); + tenants.tenantsChanged(newTenants); + Map<TenantName, Tenant> tenantsCopy = tenants.tenantsCopy(); + assertEquals(tenantsCopy.get(tenant2).getName(), tenant2); + assertEquals(tenantsCopy.get(defaultTenant).getName().value(), "default"); + assertNull(tenantsCopy.get(tenant1)); + newTenants.clear(); + tenants.tenantsChanged(newTenants); + tenantsCopy = tenants.tenantsCopy(); + assertNull(tenantsCopy.get(tenant1)); + assertNull(tenantsCopy.get(tenant2)); + assertNull(tenantsCopy.get(defaultTenant)); + newTenants.clear(); + TenantName foo = TenantName.from("foo"); + TenantName bar = TenantName.from("bar"); + newTenants.add(tenant2); + newTenants.add(foo); + newTenants.add(bar); + tenants.tenantsChanged(newTenants); + tenantsCopy = tenants.tenantsCopy(); + assertNotNull(tenantsCopy.get(tenant2)); + assertNotNull(tenantsCopy.get(foo)); + assertNotNull(tenantsCopy.get(bar)); + assertEquals(tenantsCopy.get(tenant2).getName(), tenant2); + assertEquals(tenantsCopy.get(foo).getName(), foo); + assertEquals(tenantsCopy.get(bar).getName(), bar); + } + + @Test + public void testTenantWatching() throws Exception { + TestComponentRegistry reg = new TestComponentRegistry(curator); + Tenants t = new Tenants(reg, Metrics.createTestMetrics()); + try { + assertEquals(t.tenantsCopy().get(TenantName.defaultName()).getName(), TenantName.defaultName()); + reg.getCurator().framework().create().forPath(tenants.tenantZkPath(TenantName.from("newTenant"))); + // Poll for the watcher to pick up the tenant from zk, and add it + int tries=0; + while(true) { + if (tries > 500) fail("Didn't react on watch"); + Tenant nt = t.tenantsCopy().get(TenantName.from("newTenant")); + if (nt != null) { + assertEquals(nt.getName().value(), "newTenant"); + return; + } + tries++; + Thread.sleep(100); + } + } finally { + t.close(); + } + } + + @Test + public void testTenantRedeployment() throws Exception { + MockDeployer deployer = new MockDeployer(); + Tenant tenant = tenants.tenantsCopy().get(tenant1); + ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName()); + tenant.getApplicationRepo().createPutApplicationTransaction(id, 3).commit(); + tenants.redeployApplications(deployer); + assertThat(deployer.lastDeployed, is(id)); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java new file mode 100644 index 00000000000..d202f55e38a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.FileDistributionFactory; +import com.yahoo.vespa.config.server.session.MockFileDistributionFactory; +import com.yahoo.vespa.config.server.session.SessionPreparer; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.model.VespaModelFactory; + +import java.util.Collections; +import java.util.Optional; + +/** + * @author lulf + * @since 5.1 + */ +// TODO Use a Builder to avoid so many constructors +public class TestComponentRegistry implements GlobalComponentRegistry { + + private final Curator curator; + private final ConfigCurator configCurator; + private final Metrics metrics; + private final ConfigServerDB serverDB; + private final SessionPreparer sessionPreparer; + private final ConfigserverConfig configserverConfig; + private final SuperModelGenerationCounter superModelGenerationCounter; + private final ConfigDefinitionRepo defRepo; + final TenantRequestHandlerTest.MockReloadListener reloadListener; + final MockTenantListener tenantListener; + private final PermanentApplicationPackage permanentApplicationPackage; + private final HostRegistries hostRegistries; + private final FileDistributionFactory fileDistributionFactory; + private final ModelFactoryRegistry modelFactoryRegistry; + private final Optional<Provisioner> hostProvisioner; + + public TestComponentRegistry() { this(new MockCurator()); } + + public TestComponentRegistry(Curator curator) { + this(curator, ConfigCurator.create(curator), new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))); + } + + public TestComponentRegistry(Curator curator, ConfigCurator configCurator, FileDistributionFactory fileDistributionFactory) { + this(curator, configCurator, new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))), fileDistributionFactory); + } + + public TestComponentRegistry(Curator curator, ModelFactoryRegistry modelFactoryRegistry) { + this(curator, ConfigCurator.create(curator), modelFactoryRegistry, Optional.empty()); + } + + public TestComponentRegistry(Curator curator, ConfigCurator configCurator, ModelFactoryRegistry modelFactoryRegistry) { + this(curator, configCurator, modelFactoryRegistry, Optional.empty()); + } + + public TestComponentRegistry(Curator curator, ConfigCurator configCurator, ModelFactoryRegistry modelFactoryRegistry, FileDistributionFactory fileDistributionFactory) { + this(curator, configCurator, modelFactoryRegistry, Optional.empty(), fileDistributionFactory); + } + + public TestComponentRegistry(Curator curator, ModelFactoryRegistry modelFactoryRegistry, Optional<PermanentApplicationPackage> permanentApplicationPackage) { + this(curator, ConfigCurator.create(curator), modelFactoryRegistry, permanentApplicationPackage, new MockFileDistributionFactory()); + } + + public TestComponentRegistry(Curator curator, ConfigCurator configCurator, ModelFactoryRegistry modelFactoryRegistry, Optional<PermanentApplicationPackage> permanentApplicationPackage) { + this(curator, configCurator, modelFactoryRegistry, permanentApplicationPackage, new MockFileDistributionFactory()); + } + + public TestComponentRegistry(Curator curator, ConfigCurator configCurator, ModelFactoryRegistry modelFactoryRegistry, Optional<PermanentApplicationPackage> permanentApplicationPackage, FileDistributionFactory fileDistributionFactory) { + this.curator = curator; + this.configCurator = configCurator; + metrics = Metrics.createTestMetrics(); + configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(Files.createTempDir().getAbsolutePath())); + serverDB = new ConfigServerDB(configserverConfig); + reloadListener = new TenantRequestHandlerTest.MockReloadListener(); + tenantListener = new MockTenantListener(); + this.superModelGenerationCounter = new SuperModelGenerationCounter(curator); + this.defRepo = new StaticConfigDefinitionRepo(); + this.permanentApplicationPackage = permanentApplicationPackage.orElse(new PermanentApplicationPackage(configserverConfig)); + this.hostRegistries = new HostRegistries(); + this.fileDistributionFactory = fileDistributionFactory; + this.modelFactoryRegistry = modelFactoryRegistry; + this.hostProvisioner = Optional.empty(); + sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionFactory, HostProvisionerProvider.empty(), this.permanentApplicationPackage, configserverConfig, defRepo, curator, new Zone(configserverConfig)); + } + + @Override + public Curator getCurator() { return curator; } + @Override + public ConfigCurator getConfigCurator() { return configCurator; } + @Override + public Metrics getMetrics() { return metrics; } + @Override + public ConfigServerDB getServerDB() { return serverDB; } + @Override + public SessionPreparer getSessionPreparer() { return sessionPreparer; } + @Override + public ConfigserverConfig getConfigserverConfig() { return configserverConfig; } + @Override + public TenantListener getTenantListener() { return tenantListener; } + @Override + public ReloadListener getReloadListener() { return reloadListener; } + @Override + public SuperModelGenerationCounter getSuperModelGenerationCounter() { return superModelGenerationCounter; } + @Override + public ConfigDefinitionRepo getConfigDefinitionRepo() { return defRepo; } + @Override + public PermanentApplicationPackage getPermanentApplicationPackage() { return permanentApplicationPackage; } + @Override + public HostRegistries getHostRegistries() { return hostRegistries;} + @Override + public ModelFactoryRegistry getModelFactoryRegistry() { return modelFactoryRegistry; } + + @Override + public Optional<Provisioner> getHostProvisioner() { + return hostProvisioner; + } + + @Override + public Zone getZone() { + return Zone.defaultZone(); + } + + public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java new file mode 100644 index 00000000000..2a0c78ce7a5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestConfigDefinitionRepo.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author lulf + * @since 5. + */ +public class TestConfigDefinitionRepo implements ConfigDefinitionRepo { + private final Map<ConfigDefinitionKey, ConfigDefinition> repo = new LinkedHashMap<>(); + public TestConfigDefinitionRepo() { + repo.put(new ConfigDefinitionKey(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_SCHEMA)); + repo.put(new ConfigDefinitionKey(LbServicesConfig.CONFIG_DEF_NAME, LbServicesConfig.CONFIG_DEF_NAMESPACE), + new ConfigDefinition(LbServicesConfig.CONFIG_DEF_NAME, LbServicesConfig.CONFIG_DEF_SCHEMA)); + } + + @Override + public Map<ConfigDefinitionKey, ConfigDefinition> getConfigDefinitions() { + return repo; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithCurator.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithCurator.java new file mode 100644 index 00000000000..6f68aa07634 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithCurator.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import org.apache.curator.framework.CuratorFramework; +import org.junit.Before; + +/** + * For tests that require a Curator instance + * + * @author lulf + * @since 5.16 + */ +public class TestWithCurator { + + protected ConfigCurator configCurator; + protected CuratorFramework curatorFramework; + protected Curator curator; + + @Before + public void setupZKProvider() throws Exception { + curator = new MockCurator(); + configCurator = ConfigCurator.create(curator); + curatorFramework = curator.framework(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java new file mode 100644 index 00000000000..3e5431deccd --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithRpc.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.ElkConfig; +import com.yahoo.config.provision.TenantName; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import org.junit.After; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Test running rpc server. + * + * @author lulf + * @since 5.17 + */ +public class TestWithRpc { + + protected RpcServer rpcServer; + protected MockTenantProvider tenantProvider; + protected GenerationCounter generationCounter; + private Thread t; + private Supervisor sup; + private Spec spec; + private int port; + + private List<Integer> allocatedPorts; + + @Before + public void setupRpc() throws InterruptedException { + allocatedPorts = new ArrayList<>(); + port = allocatePort(); + spec = createSpec(port); + tenantProvider = new MockTenantProvider(); + generationCounter = new MemoryGenerationCounter(); + createAndStartRpcServer(false); + } + + @After + public void teardownPortAllocator() { + for (Integer port : allocatedPorts) { + PortRangeAllocator.releasePort(port); + } + } + + protected int allocatePort() throws InterruptedException { + int port = PortRangeAllocator.findAvailablePort(); + allocatedPorts.add(port); + return port; + } + + protected void createAndStartRpcServer(boolean hostedVespa) { + rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), + new SuperModelController(generationCounter, + new TestConfigDefinitionRepo(), new ConfigserverConfig(new ConfigserverConfig.Builder()), new ElkConfig(new ElkConfig.Builder())), + Metrics.createTestMetrics(), new HostRegistries()); + rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); + t = new Thread(rpcServer); + t.start(); + sup = new Supervisor(new Transport()); + pingServer(); + } + + @After + public void stopRpc() throws InterruptedException { + rpcServer.stop(); + t.join(); + } + + private Spec createSpec(int port) { + return new Spec("tcp/localhost:" + port); + } + + private void pingServer() { + long endTime = System.currentTimeMillis() + 60_000; + Request req = new Request("ping"); + while (System.currentTimeMillis() < endTime) { + performRequest(req); + if (!req.isError() && req.returnValues().size() > 0 && req.returnValues().get(0).asInt32() == 0) { + break; + } + req = new Request("ping"); + } + assertFalse(req.isError()); + assertTrue(req.returnValues().size() > 0); + assertThat(req.returnValues().get(0).asInt32(), is(0)); + } + + protected void performRequest(Request req) { + sup.connect(spec).invokeSync(req, 120.0); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithTenant.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithTenant.java new file mode 100644 index 00000000000..3892bf505b9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestWithTenant.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.vespa.config.server.monitoring.Metrics; +import org.junit.Before; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Utility for a test using a single default tenant. + * + * @author lulf + * @since 5.35 + */ +public class TestWithTenant extends TestWithCurator { + + protected Tenants tenants; + protected Tenant tenant; + + @Before + public void setupTenant() throws Exception { + tenants = new Tenants(new TestComponentRegistry(curator), Metrics.createTestMetrics()); + tenant = tenants.defaultTenant(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TimeoutBudgetTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TimeoutBudgetTest.java new file mode 100644 index 00000000000..224a759896f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TimeoutBudgetTest.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.test.ManualClock; +import org.junit.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class TimeoutBudgetTest { + public static TimeoutBudget day() { + return new TimeoutBudget(new ManualClock(Instant.now()), Duration.ofDays(1)); + } + + @Test + public void testTimeLeft() { + ManualClock clock = new ManualClock(); + + TimeoutBudget budget = new TimeoutBudget(clock, Duration.ofMillis(7)); + assertThat(budget.timeLeft().toMillis(), is(7l)); + clock.advance(Duration.ofMillis(1)); + assertThat(budget.timeLeft().toMillis(), is(6l)); + clock.advance(Duration.ofMillis(5)); + assertThat(budget.timeLeft().toMillis(), is(1l)); + assertThat(budget.timeLeft().toMillis(), is(1l)); + clock.advance(Duration.ofMillis(1)); + assertThat(budget.timeLeft().toMillis(), is(0l)); + clock.advance(Duration.ofMillis(5)); + assertThat(budget.timeLeft().toMillis(), is(0l)); + + clock.advance(Duration.ofMillis(1)); + assertThat(budget.timesUsed(), is("[0 ms, 1 ms, 5 ms, 0 ms, 1 ms, 5 ms, total: 13 ms]")); + } + + @Test + public void testHasTimeLeft() { + ManualClock clock = new ManualClock(); + + TimeoutBudget budget = new TimeoutBudget(clock, Duration.ofMillis(7)); + assertThat(budget.hasTimeLeft(), is(true)); + clock.advance(Duration.ofMillis(1)); + assertThat(budget.hasTimeLeft(), is(true)); + clock.advance(Duration.ofMillis(5)); + assertThat(budget.hasTimeLeft(), is(true)); + assertThat(budget.hasTimeLeft(), is(true)); + clock.advance(Duration.ofMillis(1)); + assertThat(budget.hasTimeLeft(), is(false)); + clock.advance(Duration.ofMillis(5)); + assertThat(budget.hasTimeLeft(), is(false)); + + clock.advance(Duration.ofMillis(1)); + assertThat(budget.timesUsed(), is("[0 ms, 1 ms, 5 ms, 0 ms, 1 ms, 5 ms, total: 13 ms]")); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg new file mode 100644 index 00000000000..8b43ff9c793 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg @@ -0,0 +1,78 @@ +accesslog "/home/vespa/logs/vespa/foo.log" +partialsd "sd" +partialsd2 "global2" +asyncfetchocc 10 +a 0 +b 1 +c 2 +d 3 +e 4 +onlyindef 45 +listenport 13700 +rangecheck2 10 +rangecheck3 10 +kanon -78.56 +rangecheck1 10.0 +testref search/cluster.music/c0/r0/indexer.4 +testref2 some/babbel +mode BATCH +functionmodules[0] +storage[2] +storage[0].feeder[1] +storage[0].feeder[0] "test" +storage[1].id search/cluster.music/c0/r0/indexer.4 +storage[1].id2 pjatt +storage[1].feeder[2] +storage[1].feeder[0] "me" +storage[1].feeder[1] "now" +search[3] +search[0].feeder[1] +search[0].feeder[0] "foofeeder" +search[1].feeder[4] +search[1].feeder[0] "barfeeder1_1" +search[1].feeder[1] "barfeeder2" +search[1].feeder[2] "" +search[1].feeder[3] "barfeeder2_1" +search[2].feeder[2] +search[2].feeder[0] "" +search[2].feeder[1] "bazfeeder" +f[1] +f[0].a "A" +f[0].b "B" +f[0].c "C" +f[0].h "H" +f[0].f "F" +config[1] +config[0].role "rtx" +config[0].usewrapper false +config[0].id search/cluster.music/rtx/0 +routingtable[1] +routingtable[0].hop[3] +routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" +routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" +routingtable[0].hop[0].recipient[0] +routingtable[0].hop[1].name "search/cluster.music" +routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" +routingtable[0].hop[1].recipient[1] +routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" +routingtable[0].hop[2].selector "[DocumentRouteSelector]" +routingtable[0].hop[2].name "indexing" +routingtable[0].hop[2].recipient[1] +routingtable[0].hop[2].recipient[0] "search/cluster.music" +speciallog[1] +speciallog[0].filehandler.name "QueryAccessLog" +speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" +speciallog[0].filehandler.rotation "0 1 ..." +speciallog[0].cachehandler.name "QueryAccessLog" +speciallog[0].name "QueryAccessLog" +speciallog[0].type "file" +speciallog[0].cachehandler.size 1000 +rulebase[4] +rulebase[0].name "cjk" +rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n" +rulebase[1].name "common" +rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" +rulebase[2].name "egyik" +rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" +rulebase[3].name "masik" +rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg new file mode 100644 index 00000000000..927ff8a26c9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg @@ -0,0 +1,48 @@ +accesslog "/home/vespa/logs/vespa/foo.log" +partialsd "global" +partialsd2 "global2" +asyncfetchocc 10 +a 0 +b 1 +c 67 +d 89 +e 4 +onlyindef 45 +listenport 13700 +rangecheck2 10 +rangecheck3 10 +rangecheck1 10.0 +mode BATCH +functionmodules[0] +storage[0] +search[3] +search[0].feeder[1] +search[0].feeder[0] "foofeeder" +search[1].feeder[4] +search[1].feeder[0] "barfeeder1_1" +search[1].feeder[1] "sportsfeeder1" +search[1].feeder[2] "" +search[1].feeder[3] "barfeeder2_1" +search[2].feeder[2] +search[2].feeder[0] "" +search[2].feeder[1] "bazfeeder" +f[0] +config[0] +routingtable[0] +speciallog[1] +speciallog[0].filehandler.name "QueryAccessLog" +speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" +speciallog[0].filehandler.rotation "0 1 ..." +speciallog[0].cachehandler.name "QueryAccessLog" +speciallog[0].name "QueryAccessLog" +speciallog[0].type "file" +speciallog[0].cachehandler.size 1000 +rulebase[4] +rulebase[0].name "cjk" +rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n" +rulebase[1].name "common" +rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" +rulebase[2].name "egyik" +rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" +rulebase[3].name "masik" +rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar Binary files differnew file mode 100644 index 00000000000..69f6e335092 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml new file mode 100644 index 00000000000..53d7c599817 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + </admin> + + <content version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index"/> + </documents> + <nodes>> + <node hostalias="node1" distribution-key="0"/> + </nodes> + </content> +</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java new file mode 100644 index 00000000000..eefa4b6176d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationConvergenceCheckerTest.java @@ -0,0 +1,210 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Duration; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author lulf + */ +public class ApplicationConvergenceCheckerTest { + + private TenantName tenant = TenantName.from("mytenant"); + private ApplicationId appId = ApplicationId.from(tenant, ApplicationName.from("myapp"), InstanceName.from("myinstance")); + private ObjectMapper mapper = new ObjectMapper(); + private Application application; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void setup() throws IOException, SAXException, InterruptedException { + Model mockModel = new MockModel(1337); + application = new Application(mockModel, new ServerCache(), 3, Version.fromIntValues(0, 0, 0), MetricUpdater.createTestUpdater(), appId); + } + + private void assertJsonResponseEquals(HttpResponse httpResponse, String expected) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + httpResponse.render(out); + String response = out.toString(StandardCharsets.UTF_8.name()); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonResponse = mapper.readTree(response); + JsonNode jsonExpected = mapper.readTree(expected); + if (jsonExpected.equals(jsonResponse)) { + return; + } + fail("Not equal, response is '" + response + "' expected '"+ expected + "'"); + } + + @Test + public void converge() throws IOException, SAXException { + ApplicationConvergenceChecker checker = new ApplicationConvergenceChecker((client, serviceUri) -> () -> string2json("{\"config\":{\"generation\":3}}")); + checker.waitForConfigConverged(application, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(1))); + } + + @Test + public void convergeV2() throws IOException, SAXException { + ApplicationConvergenceChecker checker = new ApplicationConvergenceChecker((client, serviceUri) -> () -> string2json("{\"config\":{\"generation\":3}}")); + final HttpResponse httpResponse = checker.listConfigConvergence(application, URI.create("http://foo:234/serviceconvergence")); + assertThat(httpResponse.getStatus(), is(200)); + assertJsonResponseEquals(httpResponse, "{\"services\":[" + + "{\"port\":1337,\"host\":\"localhost\"," + + "\"url\":\"http://foo:234/serviceconvergence/localhost:1337\"," + + "\"type\":\"container\"}]," + + "\"debug\":{\"wantedVersion\":3}," + + "\"url\":\"http://foo:234/serviceconvergence\"}"); + final HttpResponse nodeHttpResponse = checker.nodeConvergenceCheck(application, "localhost:1337", URI.create("http://foo:234/serviceconvergence")); + assertThat(nodeHttpResponse.getStatus(), is(200)); + assertJsonResponseEquals(nodeHttpResponse, "{" + + "\"converged\":true," + + "\"debug\":{\"wantedGeneration\":3," + + "\"currentGeneration\":3," + + "\"host\":\"localhost:1337\"}," + + "\"url\":\"http://foo:234/serviceconvergence\"}"); + final HttpResponse hostMissingHttpResponse = checker.nodeConvergenceCheck(application, "notPresent:1337", URI.create("http://foo:234/serviceconvergence")); + assertThat(hostMissingHttpResponse.getStatus(), is(410)); + assertJsonResponseEquals(hostMissingHttpResponse, "{\"debug\":{" + + "\"problem\":\"Host:port (service) no longer part of application, refetch list of services.\"," + + "\"wantedGeneration\":3," + + "\"host\":\"notPresent:1337\"}," + + "\"url\":\"http://foo:234/serviceconvergence\"}"); + } + + // When config server constantly redeploys applications we might end up with a higher version than expected, which is OK + @Test + public void convergeGenerationIsLargerThanExpected() throws IOException, SAXException { + ApplicationConvergenceChecker checker = new ApplicationConvergenceChecker((client, serviceUri) -> () -> string2json("{\"config\":{\"generation\":4}}")); + checker.waitForConfigConverged(application, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(1))); + } + + private JsonNode string2json(String data) { + try { + return mapper.readTree(data); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void convergeFailure() throws IOException { + ApplicationConvergenceChecker checker = new ApplicationConvergenceChecker((client, serviceUri) -> () -> string2json("{\"config\":{\"generation\":2}}")); + try { + checker.waitForConfigConverged(application, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(1))); + fail("Converge should fail due to config generation not being updated"); + } catch (ConfigNotConvergedException e) { + assertThat(e.getMessage(), is("Timed out waiting for service to use config generation 3 (checking http://localhost:1337/state/v1/config), generation was 2.")); + } + final HttpResponse nodeHttpResponse = checker.nodeConvergenceCheck(application, "localhost:1337", URI.create("http://foo:234/serviceconvergence")); + assertThat(nodeHttpResponse.getStatus(), is(200)); + assertJsonResponseEquals(nodeHttpResponse, "{" + + "\"converged\":false," + + "\"debug\":{\"wantedGeneration\":3,\"currentGeneration\":2,\"host\":\"localhost:1337\" }," + + "\"url\":\"http://foo:234/serviceconvergence\"}"); + } + + @Test + public void stateApiFailure() throws IOException { + ApplicationConvergenceChecker checker = new ApplicationConvergenceChecker((client, serviceUri) -> () -> string2json("{\"config\":{}}")); + try { + checker.waitForConfigConverged(application, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(1))); + fail("Converge should fail due to config generation not being updated"); + } catch (ConfigNotConvergedException e) { + assertThat(e.getMessage(), is("Timed out waiting for service to use config generation 3 (checking http://localhost:1337/state/v1/config), could not connect.")); + } + } + + private static class MockModel implements Model { + private final int statePort; + public MockModel(int statePort) { + this.statePort = statePort; + } + + @Override + public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition targetDef, ConfigPayload override) throws IOException { + if (configKey.equals(new ConfigKey<>(ModelConfig.class, ""))) { + return createModelConfig(); + } + throw new UnsupportedOperationException(); + } + + @Override + public ConfigPayload getConfig(ConfigKey<?> configKey, InnerCNode targetDef, ConfigPayload override) throws IOException { + return getConfig(configKey, (ConfigDefinition)null, override); + } + + private ConfigPayload createModelConfig() { + ModelConfig.Builder builder = new ModelConfig.Builder(); + ModelConfig.Hosts.Builder hostBuilder = new ModelConfig.Hosts.Builder(); + hostBuilder.name("localhost"); + ModelConfig.Hosts.Services.Builder serviceBuilder = new ModelConfig.Hosts.Services.Builder(); + serviceBuilder.type("container"); + serviceBuilder.ports(new ModelConfig.Hosts.Services.Ports.Builder().number(statePort).tags("state")); + hostBuilder.services(serviceBuilder); + builder.hosts(hostBuilder); + ModelConfig config = new ModelConfig(builder); + return ConfigPayload.fromInstance(config); + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection<HostInfo> getHosts() { + throw new UnsupportedOperationException(); + } + + @Override + public Set<String> allConfigIds() { + throw new UnsupportedOperationException(); + } + + @Override + public void distributeFiles(FileDistribution fileDistribution) { + throw new UnsupportedOperationException(); + } + + @Override + public Optional<ProvisionInfo> getProvisionInfo() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationRepoTest.java new file mode 100644 index 00000000000..eea951e5c9c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationRepoTest.java @@ -0,0 +1,191 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.MockReloadHandler; +import com.yahoo.vespa.config.server.TestWithCurator; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class ApplicationRepoTest extends TestWithCurator { + + @Test + public void require_that_applications_are_read_from_zookeeper() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz:bim", Utf8.toAsciiBytes(3)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/bar:test:bim:quux", Utf8.toAsciiBytes(4)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/bario:staging:bala:bong", Utf8.toAsciiBytes(5)); + ApplicationRepo repo = createZKAppRepo(); + List<ApplicationId> applications = repo.listApplications(); + assertThat(applications.size(), is(3)); + assertThat(applications.get(0).application().value(), is("bario")); + assertThat(applications.get(1).application().value(), is("bar")); + assertThat(applications.get(2).application().value(), is("foo")); + assertThat(repo.getSessionIdForApplication(applications.get(0)), is(5l)); + assertThat(repo.getSessionIdForApplication(applications.get(1)), is(4l)); + assertThat(repo.getSessionIdForApplication(applications.get(2)), is(3l)); + } + + @Test + public void require_that_legacy_application_ids_are_rewritten() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:default:baz:bim", Utf8.toAsciiBytes(3)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/bar:test:bim:quux", Utf8.toAsciiBytes(4)); + ApplicationRepo repo = createZKAppRepo(); + List<ApplicationId> applications = repo.listApplications(); + assertThat(applications.size(), is(2)); + assertThat(applications.get(0).application().value(), is("bar")); + assertThat(applications.get(1).application().value(), is("foo")); + assertThat(applications.get(0).instance().value(), is("quux")); + assertThat(applications.get(1).instance().value(), is("bim")); + assertNotNull(curatorFramework.checkExists().forPath("/mytenant:foo:bim")); + assertNotNull(curatorFramework.checkExists().forPath("/mytenant:foo:bim")); + assertThat(repo.getSessionIdForApplication(applications.get(0)), is(4l)); + assertThat(repo.getSessionIdForApplication(applications.get(1)), is(3l)); + } + + @Test + public void require_that_legacy_application_ids_are_ignored() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:default:baz:bim", Utf8.toAsciiBytes(3)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:prod:baz:bim", Utf8.toAsciiBytes(3)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/bar:test:bim:quux", Utf8.toAsciiBytes(4)); + ApplicationRepo repo = createZKAppRepo(); + List<ApplicationId> applications = repo.listApplications(); + assertThat(applications.size(), is(2)); + assertThat(repo.getSessionIdForApplication(applications.get(0)), is(4l)); + assertThat(repo.getSessionIdForApplication(applications.get(1)), is(3l)); + } + + @Test + public void require_that_invalid_entries_are_skipped() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz:bim"); + curatorFramework.create().creatingParentsIfNeeded().forPath("/invalid"); + ApplicationRepo repo = createZKAppRepo(); + List<ApplicationId> applications = repo.listApplications(); + assertThat(applications.size(), is(1)); + assertThat(applications.get(0).application().value(), is("foo")); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_requesting_session_for_unknown_application_throws_exception() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz:bim"); + ApplicationRepo repo = createZKAppRepo(); + repo.getSessionIdForApplication(new ApplicationId.Builder() + .tenant("exist") + .applicationName("tenant").instanceName("here").build()); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_requesting_session_for_empty_application_throws_exception() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:dev:baz:bim"); + ApplicationRepo repo = createZKAppRepo(); + repo.getSessionIdForApplication(new ApplicationId.Builder() + .tenant("tenant") + .applicationName("foo").instanceName("bim").build()); + } + + @Test + public void require_that_application_ids_can_be_written() throws Exception { + ApplicationRepo repo = createZKAppRepo(); + repo.createPutApplicationTransaction(createAppplicationId("myapp"), 3l).commit(); + String path = "/mytenant:myapp:myinst"; + assertTrue(curatorFramework.checkExists().forPath(path) != null); + assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("3")); + repo.createPutApplicationTransaction(createAppplicationId("myapp"), 5l).commit(); + assertTrue(curatorFramework.checkExists().forPath(path) != null); + assertThat(Utf8.toString(curatorFramework.getData().forPath(path)), is("5")); + } + + @Test + public void require_that_application_ids_can_be_deleted() throws Exception { + ApplicationRepo repo = createZKAppRepo(); + ApplicationId id1 = createAppplicationId("myapp"); + ApplicationId id2 = createAppplicationId("myapp2"); + repo.createPutApplicationTransaction(id1, 1).commit(); + repo.createPutApplicationTransaction(id2, 1).commit(); + assertThat(repo.listApplications().size(), is(2)); + repo.deleteApplication(id1); + assertThat(repo.listApplications().size(), is(1)); + repo.deleteApplication(id2); + assertThat(repo.listApplications().size(), is(0)); + repo.deleteApplication(id2); + assertThat(repo.listApplications().size(), is(0)); + } + + @Test + public void require_that_repos_behave_similarly() throws Exception { + ApplicationRepo zkRepo = createZKAppRepo(); + ApplicationRepo memRepo = new MemoryApplicationRepo(); + for (ApplicationRepo repo : Arrays.asList(zkRepo, memRepo)) { + ApplicationId id1 = createAppplicationId("myapp"); + ApplicationId id2 = createAppplicationId("myapp2"); + repo.createPutApplicationTransaction(id1, 4).commit(); + repo.createPutApplicationTransaction(id2, 5).commit(); + List<ApplicationId> lst = repo.listApplications(); + Collections.sort(lst); + assertThat(lst.size(), is(2)); + assertThat(lst.get(0).application(), is(id1.application())); + assertThat(lst.get(1).application(), is(id2.application())); + assertThat(repo.getSessionIdForApplication(id1), is(4l)); + assertThat(repo.getSessionIdForApplication(id2), is(5l)); + repo.createPutApplicationTransaction(id1, 6).commit(); + lst = repo.listApplications(); + Collections.sort(lst); + assertThat(lst.size(), is(2)); + assertThat(lst.get(0).application(), is(id1.application())); + assertThat(lst.get(1).application(), is(id2.application())); + assertThat(repo.getSessionIdForApplication(id1), is(6l)); + assertThat(repo.getSessionIdForApplication(id2), is(5l)); + repo.deleteApplication(id1); + assertThat(repo.listApplications().size(), is(1)); + repo.deleteApplication(id2); + assertThat(repo.listApplications().size(), is(0)); + repo.deleteApplication(id2); + } + } + + @Test + public void require_that_reload_handler_is_called_when_apps_are_removed() throws Exception { + curatorFramework.create().creatingParentsIfNeeded().forPath("/foo:test:baz:bim", Utf8.toAsciiBytes(3)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/bar:dev:bim:quux", Utf8.toAsciiBytes(4)); + curatorFramework.create().creatingParentsIfNeeded().forPath("/bario:staging:bala:bong", Utf8.toAsciiBytes(5)); + MockReloadHandler reloadHandler = new MockReloadHandler(); + ApplicationRepo repo = createZKAppRepo(reloadHandler); + assertNull(reloadHandler.lastRemoved); + repo.deleteApplication(new ApplicationId.Builder() + .tenant("mytenant") + .applicationName("bar").instanceName("quux").build()); + long endTime = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { + Thread.sleep(100); + } + assertNotNull(reloadHandler.lastRemoved); + assertThat(reloadHandler.lastRemoved.serializedForm(), is("mytenant:bar:quux")); + } + + private ApplicationRepo createZKAppRepo() { + return createZKAppRepo(new MockReloadHandler()); + } + + private ApplicationRepo createZKAppRepo(MockReloadHandler reloadHandler) { + return ZKApplicationRepo.create(curator, Path.createRoot(), reloadHandler, TenantName.from("mytenant")); + } + + private static ApplicationId createAppplicationId(String name) { + return new ApplicationId.Builder() + .tenant("mytenant") + .applicationName(name).instanceName("myinst").build(); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java new file mode 100644 index 00000000000..87313ed5b42 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java @@ -0,0 +1,155 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.vespa.config.server.ModelStub; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.UnknownConfigDefinitionException; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1.14 + */ +public class ApplicationTest { + + @Test + public void testThatApplicationIsInitialized() throws IOException, SAXException { + ApplicationId appId = ApplicationId.from(TenantName.defaultName(), + ApplicationName.from("foobar"), InstanceName.defaultName()); + ServerCache cache = new ServerCache(); + Version vespaVersion = Version.fromIntValues(1, 2, 3); + Application app = new Application(new ModelStub(), cache, 1337, vespaVersion, MetricUpdater.createTestUpdater(), appId); + assertThat(app.getApplicationGeneration(), is(1337l)); + assertNotNull(app.getModel()); + assertThat(app.getCache(), is(cache)); + assertThat(app.getName(), is("foobar")); + assertThat(app.getVespaVersion(), is(vespaVersion)); + assertThat(app.toString(), is("application 'foobar', generation 1337, vespa version 1.2.3")); + } + + private static final String[] emptySchema = new String[0]; + + private Application handler; + + @Before + public void setupHandler() throws IOException, SAXException { + File testApp = new File("src/test/apps/app"); + ServerCache cache = createCacheAndAddContent(); + VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(testApp)); + final ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("foo").build(); + handler = new Application(model, cache, 1, Version.fromIntValues(1, 2, 3), + new MetricUpdater(Metrics.createTestMetrics(), Metrics.createDimensions(applicationId)), applicationId); + } + + private static ServerCache createCacheAndAddContent() { + ServerCache cache = new ServerCache(); + + final ConfigDefinitionKey key = new ConfigDefinitionKey(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE); + com.yahoo.vespa.config.buildergen.ConfigDefinition def = getDef(key, SimpletypesConfig.CONFIG_DEF_SCHEMA); + // TODO Why do we have to use empty def md5 here? + cache.addDef(key, def); + + final ConfigDefinitionKey key2 = new ConfigDefinitionKey(SlobroksConfig.CONFIG_DEF_NAME, SlobroksConfig.CONFIG_DEF_NAMESPACE); + com.yahoo.vespa.config.buildergen.ConfigDefinition def2 = getDef(key2, SlobroksConfig.CONFIG_DEF_SCHEMA); + cache.addDef(key2, def2); + + final ConfigDefinitionKey key3 = new ConfigDefinitionKey(LogdConfig.CONFIG_DEF_NAME, LogdConfig.CONFIG_DEF_NAMESPACE); + com.yahoo.vespa.config.buildergen.ConfigDefinition def3 = getDef(key3, LogdConfig.CONFIG_DEF_SCHEMA); + cache.addDef(key3, def3); + + return cache; + } + + private static com.yahoo.vespa.config.buildergen.ConfigDefinition getDef(ConfigDefinitionKey key, String[] schema) { + return new com.yahoo.vespa.config.buildergen.ConfigDefinition(key.getName(), schema); + } + + @Test(expected = UnknownConfigDefinitionException.class) + public void require_that_def_file_must_exist() { + handler.resolveConfig(createRequest("unknown", "namespace", "a", emptySchema)); + } + + @Test + public void require_that_known_config_defs_are_found() throws IOException, SAXException { + handler.resolveConfig(createSimpleConfigRequest(emptySchema)); + } + + @Test + public void require_that_build_config_can_be_resolved() throws IOException, SAXException { + List<String> payload = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)).getLegacyPayload(); + assertTrue(payload.get(1).contains("host")); + } + + @Test + public void require_that_non_existent_fields_in_schema_is_skipped() throws IOException, SAXException { + // Ask for config without schema and check that we get correct default value back + List<String> payload = handler.resolveConfig(createSimpleConfigRequest(emptySchema)).getLegacyPayload(); + assertThat(payload.get(0), is("boolval false")); + // Ask for config with wrong schema + String[] schema = new String[1]; + schema[0] = "boolval bool default=true"; // changed to be true, original is false + payload = handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)).getLegacyPayload(); + assertThat(payload.size(), is(1)); + assertThat(payload.get(0), is("boolval true")); + } + + @Test + public void require_that_configs_are_cached() { + ConfigResponse response = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)); + assertNotNull(response); + ConfigResponse cached_response = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)); + assertNotNull(cached_response); + assertTrue(response == cached_response); + } + + private static GetConfigRequest createRequest(String name, String namespace, String defMd5, String[] schema, String configId) { + Request request = JRTClientConfigRequestV3. + createWithParams(new ConfigKey<>(name, configId, namespace, defMd5, null), DefContent.fromArray(schema), + "fromHost", "", 0, 100, Trace.createDummy(), CompressionType.UNCOMPRESSED, + Optional.empty()).getRequest(); + return JRTServerConfigRequestV3.createFromRequest(request); + } + + private static GetConfigRequest createRequest(String name, String namespace, String defMd5, String[] schema) { + return createRequest(name, namespace, defMd5, schema, "admin/model"); + } + + private static GetConfigRequest createSimpleConfigRequest(String[] schema) { + return createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, SimpletypesConfig.CONFIG_DEF_MD5, schema); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryApplicationRepo.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryApplicationRepo.java new file mode 100644 index 00000000000..c725775e467 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MemoryApplicationRepo.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.session.DummyTransaction; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * In memory {@link ApplicationRepo} to be used when testing. + * + * @author lulf + * @since 5.1 + */ +public class MemoryApplicationRepo implements ApplicationRepo { + private final Map<ApplicationId, Long> applications = new LinkedHashMap<>(); + private boolean isOpen = true; + + @Override + public List<ApplicationId> listApplications() { + List<ApplicationId> lst = new ArrayList<>(); + lst.addAll(applications.keySet()); + return lst; + } + + @Override + public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { + return new DummyTransaction().add((DummyTransaction.RunnableOperation) () -> { + applications.put(applicationId, sessionId); + }); + } + + @Override + public long getSessionIdForApplication(ApplicationId id) { + if (applications.containsKey(id)) { + return applications.get(id); + } + return 0; + } + + @Override + public void deleteApplication(ApplicationId id) { + applications.remove(id); + } + + @Override + public void close() { + isOpen = false; + } + + public boolean isOpen() { + return isOpen; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackageTest.java new file mode 100644 index 00000000000..a710384701f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackageTest.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.cloud.config.ConfigserverConfig; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.15 + */ +public class PermanentApplicationPackageTest { + @Test + public void testNonexistingApplication() { + PermanentApplicationPackage permanentApplicationPackage = new PermanentApplicationPackage(new ConfigserverConfig(new ConfigserverConfig.Builder().applicationDirectory("_no_such_dir"))); + assertFalse(permanentApplicationPackage.applicationPackage().isPresent()); + } + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testExistingApplication() throws IOException { + File tmpDir = folder.newFolder(); + PermanentApplicationPackage permanentApplicationPackage = new PermanentApplicationPackage(new ConfigserverConfig(new ConfigserverConfig.Builder().applicationDirectory(tmpDir.getAbsolutePath()))); + assertTrue(permanentApplicationPackage.applicationPackage().isPresent()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java new file mode 100644 index 00000000000..c99561fe7d2 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author geirst + * @since 5.44 + */ +public class ConfigChangeActionsBuilder { + + private final List<ConfigChangeAction> actions = new ArrayList<>(); + + private static ServiceInfo createService(String clusterName, String clusterType, String serviceType, String serviceName) { + return new ServiceInfo(serviceName, serviceType, null, + ImmutableMap.of("clustername", clusterName, "clustertype", clusterType), + serviceType + "/" + serviceName, "hostname"); + } + + public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName) { + actions.add(new MockRestartAction(message, + Arrays.asList(createService(clusterName, clusterType, serviceType, serviceName)))); + return this; + } + + public ConfigChangeActionsBuilder refeed(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { + actions.add(new MockRefeedAction(name, + allowed, + message, + Arrays.asList(createService(clusterName, "myclustertype", "myservicetype", serviceName)), documentType)); + return this; + } + + public ConfigChangeActions build() { + return new ConfigChangeActions(actions); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java new file mode 100644 index 00000000000..84c69fef3a1 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsSlimeConverterTest.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static com.yahoo.vespa.config.server.configchange.Utils.*; + +/** + * @author geirst + * @since 5.44 + */ +public class ConfigChangeActionsSlimeConverterTest { + + private static String toJson(ConfigChangeActions actions) throws IOException { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + new ConfigChangeActionsSlimeConverter(actions).toSlime(root); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + new JsonFormat(false).encode(outputStream, slime); + return outputStream.toString(); + } + + @Test + public void json_representation_of_empty_actions() throws IOException { + ConfigChangeActions actions = new ConfigChangeActionsBuilder().build(); + assertEquals( "{\n" + + " \"configChangeActions\": {\n" + + " \"restart\": [\n" + + " ],\n" + + " \"refeed\": [\n" + + " ]\n" + + " }\n" + + "}\n", + toJson(actions)); + } + + @Test + public void json_representation_of_restart_actions() throws IOException { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME_2). + restart(CHANGE_MSG_2, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG_2, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME_2).build(); + assertEquals("{\n" + + " \"configChangeActions\": {\n" + + " \"restart\": [\n" + + " {\n" + + " \"clusterName\": \"foo\",\n" + + " \"clusterType\": \"search\",\n" + + " \"serviceType\": \"searchnode\",\n" + + " \"messages\": [\n" + + " \"change\",\n" + + " \"other change\"\n" + + " ],\n" + + " \"services\": [\n" + + " {\n" + + " \"serviceName\": \"baz\",\n" + + " \"serviceType\": \"searchnode\",\n" + + " \"configId\": \"searchnode/baz\",\n" + + " \"hostName\": \"hostname\"\n" + + " },\n" + + " {\n" + + " \"serviceName\": \"qux\",\n" + + " \"serviceType\": \"searchnode\",\n" + + " \"configId\": \"searchnode/qux\",\n" + + " \"hostName\": \"hostname\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"refeed\": [\n" + + " ]\n" + + " }\n" + + "}\n", + toJson(actions)); + } + + @Test + public void json_representation_of_refeed_actions() throws IOException { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + refeed(CHANGE_ID, true, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_TYPE). + refeed(CHANGE_ID_2, false, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_TYPE).build(); + assertEquals("{\n" + + " \"configChangeActions\": {\n" + + " \"restart\": [\n" + + " ],\n" + + " \"refeed\": [\n" + + " {\n" + + " \"name\": \"change-id\",\n" + + " \"allowed\": true,\n" + + " \"documentType\": \"music\",\n" + + " \"clusterName\": \"foo\",\n" + + " \"messages\": [\n" + + " \"change\"\n" + + " ],\n" + + " \"services\": [\n" + + " {\n" + + " \"serviceName\": \"searchnode\",\n" + + " \"serviceType\": \"myservicetype\",\n" + + " \"configId\": \"myservicetype/searchnode\",\n" + + " \"hostName\": \"hostname\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"name\": \"other-change-id\",\n" + + " \"allowed\": false,\n" + + " \"documentType\": \"book\",\n" + + " \"clusterName\": \"foo\",\n" + + " \"messages\": [\n" + + " \"change\"\n" + + " ],\n" + + " \"services\": [\n" + + " {\n" + + " \"serviceName\": \"searchnode\",\n" + + " \"serviceType\": \"myservicetype\",\n" + + " \"configId\": \"myservicetype/searchnode\",\n" + + " \"hostName\": \"hostname\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n", + toJson(actions)); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockConfigChangeAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockConfigChangeAction.java new file mode 100644 index 00000000000..88be73bedba --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockConfigChangeAction.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.List; + +/** + * @author geirst + * @since 5.44 + */ +public abstract class MockConfigChangeAction implements ConfigChangeAction { + + private final String message; + private final List<ServiceInfo> services; + + protected MockConfigChangeAction(String message, List<ServiceInfo> services) { + this.message = message; + this.services = services; + } + + @Override + public String getMessage() { + return message; + } + + @Override + public List<ServiceInfo> getServices() { + return services; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java new file mode 100644 index 00000000000..9043b90b15f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ConfigChangeRefeedAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.List; + +/** + * @author geirst + * @since 5.44 + */ +public class MockRefeedAction extends MockConfigChangeAction implements ConfigChangeRefeedAction { + + private final String name; + private final boolean allowed; + private final String documentType; + + public MockRefeedAction(String name, boolean allowed, String message, List<ServiceInfo> services, String documentType) { + super(message, services); + this.name = name; + this.allowed = allowed; + this.documentType = documentType; + } + + @Override + public String name() { return name; } + + @Override + public boolean allowed() { return allowed; } + + @Override + public String getDocumentType() { return documentType; } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java new file mode 100644 index 00000000000..1d546ae65c0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import com.yahoo.config.model.api.ConfigChangeRestartAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.List; + +/** + * @author geirst + * @since 5.44 + */ +public class MockRestartAction extends MockConfigChangeAction implements ConfigChangeRestartAction { + public MockRestartAction(String message, List<ServiceInfo> services) { + super(message, services); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java new file mode 100644 index 00000000000..cf4dda7d090 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsFormatterTest.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static com.yahoo.vespa.config.server.configchange.Utils.*; + +/** + * @author geirst + * @since 5.44 + */ +public class RefeedActionsFormatterTest { + + @Test + public void formatting_of_single_action() { + RefeedActions actions = new ConfigChangeActionsBuilder(). + refeed(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + build().getRefeedActions(); + assertEquals("change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + " 1) change\n", + new RefeedActionsFormatter(actions).format()); + } + + @Test + public void formatting_of_multiple_actions() { + RefeedActions actions = new ConfigChangeActionsBuilder(). + refeed(CHANGE_ID, false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID_2, false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID_2, true, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed(CHANGE_ID, false, CHANGE_MSG_2, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + build().getRefeedActions(); + assertEquals("change-id: Consider removing data and re-feed document type 'book' in cluster 'foo' because:\n" + + " 1) other change\n" + + "change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + " 1) change\n" + + " 2) other change\n" + + "other-change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + " 1) other change\n" + + "(allowed) other-change-id: Consider removing data and re-feed document type 'music' in cluster 'foo' because:\n" + + " 1) other change\n", + new RefeedActionsFormatter(actions).format()); + } + +}
\ No newline at end of file diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java new file mode 100644 index 00000000000..f1bb48eef21 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RefeedActionsTest.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertThat; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static com.yahoo.vespa.config.server.configchange.Utils.*; + +/** + * @author geirst + * @since 5.44 + */ +public class RefeedActionsTest { + + private String toString(RefeedActions.Entry entry) { + StringBuilder builder = new StringBuilder(); + builder.append(entry.getDocumentType() + "." + entry.getClusterName() + ":"); + builder.append(entry.getServices().stream(). + map(service -> service.getServiceName()). + sorted(). + collect(Collectors.joining(",", "[", "]"))); + builder.append(entry.getMessages().stream(). + collect(Collectors.joining(",", "[", "]"))); + return builder.toString(); + } + + @Test + public void action_with_multiple_reasons() { + List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed("change-id", false, CHANGE_MSG_2, DOC_TYPE, CLUSTER, SERVICE_NAME). + build().getRefeedActions().getEntries(); + assertThat(entries.size(), is(1)); + assertThat(toString(entries.get(0)), equalTo("music.foo:[baz][change,other change]")); + } + + @Test + public void actions_with_multiple_services() { + List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME_2). + build().getRefeedActions().getEntries(); + assertThat(entries.size(), is(1)); + assertThat(toString(entries.get(0)), equalTo("music.foo:[baz,qux][change]")); + } + + @Test + public void actions_with_multiple_document_types() { + List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE_2, CLUSTER, SERVICE_NAME). + build().getRefeedActions().getEntries(); + assertThat(entries.size(), is(2)); + assertThat(toString(entries.get(0)), equalTo("book.foo:[baz][change]")); + assertThat(toString(entries.get(1)), equalTo("music.foo:[baz][change]")); + } + + @Test + public void actions_with_multiple_clusters() { + List<RefeedActions.Entry> entries = new ConfigChangeActionsBuilder(). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER, SERVICE_NAME). + refeed("change-id", false, CHANGE_MSG, DOC_TYPE, CLUSTER_2, SERVICE_NAME). + build().getRefeedActions().getEntries(); + assertThat(entries.size(), is(2)); + assertThat(toString(entries.get(0)), equalTo("music.bar:[baz][change]")); + assertThat(toString(entries.get(1)), equalTo("music.foo:[baz][change]")); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatterTest.java new file mode 100644 index 00000000000..3363e022034 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatterTest.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import org.junit.Test; + +import static org.junit.Assert.assertThat; +import static org.hamcrest.CoreMatchers.equalTo; +import static com.yahoo.vespa.config.server.configchange.Utils.*; + +/** + * @author geirst + * @since 5.44 + */ +public class RestartActionsFormatterTest { + + @Test + public void formatting_of_single_action() { + RestartActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + build().getRestartActions(); + assertThat(new RestartActionsFormatter(actions).format(), + equalTo("In cluster 'foo' of type 'search':\n" + + " Restart services of type 'searchnode' because:\n" + + " 1) change\n")); + } + + @Test + public void formatting_of_multiple_actions() { + RestartActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG_2, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG, CLUSTER_2, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + build().getRestartActions(); + assertThat(new RestartActionsFormatter(actions).format(), + equalTo("In cluster 'bar' of type 'search':\n" + + " Restart services of type 'searchnode' because:\n" + + " 1) change\n" + + "In cluster 'foo' of type 'search':\n" + + " Restart services of type 'searchnode' because:\n" + + " 1) change\n" + + " 2) other change\n")); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java new file mode 100644 index 00000000000..4d866e7e2f6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertThat; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static com.yahoo.vespa.config.server.configchange.Utils.*; + +/** + * @author geirst + * @since 5.44 + */ +public class RestartActionsTest { + + private String toString(RestartActions.Entry entry) { + StringBuilder builder = new StringBuilder(); + builder.append(entry.getClusterType() + "." + entry.getClusterName() + "." + entry.getServiceType() + ":"); + builder.append(entry.getServices().stream(). + map(service -> service.getServiceName()). + sorted(). + collect(Collectors.joining(",", "[", "]"))); + builder.append(entry.getMessages().stream(). + collect(Collectors.joining(",", "[", "]"))); + return builder.toString(); + } + + @Test + public void actions_with_multiple_reasons() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG_2, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME).build(); + List<RestartActions.Entry> entries = actions.getRestartActions().getEntries(); + assertThat(entries.size(), is(1)); + assertThat(toString(entries.get(0)), equalTo("search.foo.searchnode:[baz][change,other change]")); + } + + @Test + public void actions_with_same_service_type() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME_2).build(); + List<RestartActions.Entry> entries = actions.getRestartActions().getEntries(); + assertThat(entries.size(), is(1)); + assertThat(toString(entries.get(0)), equalTo("search.foo.searchnode:[baz,qux][change]")); + } + + @Test + public void actions_with_multiple_service_types() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE_2, SERVICE_NAME).build(); + List<RestartActions.Entry> entries = actions.getRestartActions().getEntries(); + assertThat(entries.size(), is(2)); + assertThat(toString(entries.get(0)), equalTo("search.foo.distributor:[baz][change]")); + assertThat(toString(entries.get(1)), equalTo("search.foo.searchnode:[baz][change]")); + } + + @Test + public void actions_with_multiple_clusters_of_same_type() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG, CLUSTER_2, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME).build(); + List<RestartActions.Entry> entries = actions.getRestartActions().getEntries(); + assertThat(entries.size(), is(2)); + assertThat(toString(entries.get(0)), equalTo("search.bar.searchnode:[baz][change]")); + assertThat(toString(entries.get(1)), equalTo("search.foo.searchnode:[baz][change]")); + } + + @Test + public void actions_with_multiple_clusters_of_different_type() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder(). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME). + restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE_2, SERVICE_TYPE, SERVICE_NAME).build(); + List<RestartActions.Entry> entries = actions.getRestartActions().getEntries(); + assertThat(entries.size(), is(2)); + assertThat(toString(entries.get(0)), equalTo("content.foo.searchnode:[baz][change]")); + assertThat(toString(entries.get(1)), equalTo("search.foo.searchnode:[baz][change]")); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java new file mode 100644 index 00000000000..66772488c60 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/Utils.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.configchange; + +/** + * @author geirst + * @since 5.44 + */ +public class Utils { + + final static String CHANGE_ID = "change-id"; + final static String CHANGE_ID_2 = "other-change-id"; + final static String CHANGE_MSG = "change"; + final static String CHANGE_MSG_2 = "other change"; + final static String DOC_TYPE = "music"; + final static String DOC_TYPE_2 = "book"; + final static String CLUSTER = "foo"; + final static String CLUSTER_2 = "bar"; + final static String CLUSTER_TYPE = "search"; + final static String CLUSTER_TYPE_2 = "content"; + final static String SERVICE_TYPE = "searchnode"; + final static String SERVICE_TYPE_2 = "distributor"; + final static String SERVICE_NAME = "baz"; + final static String SERVICE_NAME_2 = "qux"; +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def new file mode 100644 index 00000000000..983f16ac932 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def @@ -0,0 +1,60 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +storage[].feeder[] string +search[].feeder[] string +storage[].id reference +storage[].id2 reference +accesslog string default="" +asyncfetchocc int default=0 +a int default=0 +b int default=0 +functionmodules[] string restart +c int default=0 +d int default=0 +e int default=0 +kanon double +testref reference +testref2 reference +onlyindef int +model string +f[].b string +f[].a string +f[].c string +f[].f string +f[].h string + +# The name of predefined roles. +config[].role string + +## Reference to the config to be used by the role. +config[].id reference + +## Wether the NC should start the corresponding role using the +## slavewrapper utility application or not. +config[].usewrapper bool default=false + +routingtable[].hop[].name string +routingtable[].hop[].selector string +routingtable[].hop[].recipient[] string +listenport int default=13700 + +speciallog[].name string +speciallog[].type string +speciallog[].filehandler.name string default="THEDEF" +speciallog[].filehandler.pattern string default="THEDEF.%Y%m%d%H%M%S" +speciallog[].filehandler.rotation string default="THEDEF0 60 ..." +speciallog[].cachehandler.name string default="THEDEF" +speciallog[].cachehandler.size int default=1000 + +partialsd string default = "def" +partialsd2 string default = "def2" + +rulebase[].name string +rulebase[].isdefault bool default=false +rulebase[].automata string default="" +rulebase[].rules string + +mode enum { BATCH, REALTIME, INCREMENTAL} default=BATCH +rangecheck1 double default=10 range=[-1.6,54] +rangecheck2 int default=10 range=[1,100] +rangecheck3 long default=10 range=[9,13] diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def new file mode 100644 index 00000000000..706a5c4c4f6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +gaff int default=0 +usercfgwithid int diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def new file mode 100644 index 00000000000..61a5fa045d1 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +foo string +gaz int diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def new file mode 100644 index 00000000000..2173d72635b --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +classes[].id int +classes[].name string +classes[].fields[].name string +classes[].fields[].type string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def new file mode 100644 index 00000000000..69cbeb31342 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +thestring string default="g" +theint int default=6 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def new file mode 100644 index 00000000000..ba11bca16cd --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def @@ -0,0 +1,4 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +# this one will be implicit, no cfg +fo int default=-45 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def new file mode 100644 index 00000000000..699649ffebf --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=4 +rec int +ursive string +national int +teatern int +ilscript[].name string +ilscript[].doctype string +ilscript[].content[] string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def new file mode 100644 index 00000000000..09b5d6718e6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def @@ -0,0 +1,16 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 + +# Which directory to find spool files in. +directory string default="/home/vespa/var/spool/vespa" + +# If true, move successfully processed files to <directory>/success +keepsuccess bool default=false + +# Trace level on error messages from messagebus +tracelevel int default=5 + +# Which parsers to use and config for each of them. +parsers[].classname string +parsers[].parameters[].key string +parsers[].parameters[].value string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java new file mode 100644 index 00000000000..fbd4f791f83 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostFilter; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.ProvisionLogger; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.path.Path; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.server.TestWithTenant; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class HostedDeployTest extends TestWithTenant { + + private static final Path appPath = Path.createRoot().append("testapp"); + private File testApp = new File("src/test/apps/hosted/"); + private Path tenantPath = appPath; + + @Test + public void testRedeploy() throws InterruptedException, IOException { + ApplicationId id = deployApp(); + + Deployer deployer = new Deployer(tenants, HostProvisionerProvider.withProvisioner(createHostProvisioner()), + new ConfigserverConfig(new ConfigserverConfig.Builder()), curator); + + Optional<com.yahoo.config.provision.Deployment> deployment = deployer.deployFromLocalActive(id, Duration.ofSeconds(60)); + assertTrue(deployment.isPresent()); + deployment.get().prepare(); + deployment.get().activate(); + } + + /** + * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. + */ + private ApplicationId deployApp() throws InterruptedException, IOException { + LocalSession session = tenant.getSessionFactory().createSession(testApp, "default", new SilentDeployLogger(), new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60))); + ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from("myapp"), InstanceName.defaultName()); + session.prepare(new SilentDeployLogger(), new PrepareParams(new ConfigserverConfig(new ConfigserverConfig.Builder())).applicationId(id), Optional.empty(), tenantPath); + session.createActivateTransaction().commit(); + tenant.getLocalSessionRepo().addSession(session); + return id; + } + + private Provisioner createHostProvisioner() { + return new ProvisionerAdapter(new InMemoryProvisioner(true, "host0", "host1", "host2")); + } + + private static class ProvisionerAdapter implements Provisioner { + + private final HostProvisioner hostProvisioner; + + public ProvisionerAdapter(HostProvisioner hostProvisioner) { + this.hostProvisioner = hostProvisioner; + } + + @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return hostProvisioner.prepare(cluster, capacity, groups, logger); + } + + @Override + public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { + // noop + } + + @Override + public void removed(ApplicationId application) { + // noop + } + + @Override + public void restart(ApplicationId application, HostFilter filter) { + // noop + } + + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java new file mode 100644 index 00000000000..0dd01404109 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployment; + +import java.time.Duration; +import java.util.Optional; + +/** + * @author lulf + */ +public class MockDeployer implements com.yahoo.config.provision.Deployer { + public ApplicationId lastDeployed; + + @Override + public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout) { + lastDeployed = application; + return Optional.empty(); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java new file mode 100644 index 00000000000..6175e8cf1fc --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.TestWithTenant; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests redeploying of an already existing application. + * + * @author bratseth + */ +public class RedeployTest extends TestWithTenant { + + private static final Path appPath = Path.createRoot().append("testapp"); + private File testApp = new File("src/test/apps/app"); + private Path tenantPath = appPath; + + @Test + public void testRedeploy() throws InterruptedException, IOException { + ApplicationId id = deployApp(); + + Deployer deployer = new Deployer(tenants, HostProvisionerProvider.empty(), + new ConfigserverConfig(new ConfigserverConfig.Builder()), curator); + + Optional<com.yahoo.config.provision.Deployment> deployment = deployer.deployFromLocalActive(id, Duration.ofSeconds(60)); + assertTrue(deployment.isPresent()); + long activeSessionIdBefore = tenant.getLocalSessionRepo().getActiveSession(id).getSessionId(); + assertEquals(id, tenant.getLocalSessionRepo().getSession(activeSessionIdBefore).getApplicationId()); + deployment.get().prepare(); + deployment.get().activate(); + long activeSessionIdAfter = tenant.getLocalSessionRepo().getActiveSession(id).getSessionId(); + assertEquals(activeSessionIdAfter, activeSessionIdBefore + 1); + assertEquals(id, tenant.getLocalSessionRepo().getSession(activeSessionIdAfter).getApplicationId()); + } + + /** No deploYMENT is done because there isn't a local active session. */ + @Test + public void testNoRedeploy() { + ApplicationId id = ApplicationId.from(TenantName.from("default"), + ApplicationName.from("default"), + InstanceName.from("default")); + + Deployer deployer = new Deployer(tenants, HostProvisionerProvider.empty(), + new ConfigserverConfig(new ConfigserverConfig.Builder()), curator); + + assertFalse(deployer.deployFromLocalActive(id, Duration.ofSeconds(60)).isPresent()); + } + + /** + * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. + */ + private ApplicationId deployApp() throws InterruptedException, IOException { + LocalSession session = tenant.getSessionFactory().createSession(testApp, "default", new SilentDeployLogger(), new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60))); + ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from("myapp"), InstanceName.defaultName()); + session.prepare(new SilentDeployLogger(), new PrepareParams(new ConfigserverConfig(new ConfigserverConfig.Builder())).applicationId(id), Optional.empty(), tenantPath); + session.createActivateTransaction().commit(); + tenant.getLocalSessionRepo().addSession(session); + return id; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java new file mode 100644 index 00000000000..03cb5ada8ad --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.*; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.TestWithCurator; +import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + + +/** + * Unit tests for ZooKeeperClient. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public class ZooKeeperClientTest extends TestWithCurator { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private ConfigCurator zk; + private String appPath = "/1"; + + @Before + public void setupZK() throws IOException { + this.zk = ConfigCurator.create(curator); + ZooKeeperClient zkc = new ZooKeeperClient(zk, new BaseDeployLogger(), true, Path.fromString(appPath)); + ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(new File("src/test/apps/zkfeed"), new DeployData("foo", "/bar/baz", "appName", 1345l, 3l, 2l)); + Map<Version, FileRegistry> fileRegistries = createFileRegistries(); + app.writeMetaData(); + zkc.setupZooKeeper(); + zkc.feedZooKeeper(app); + zkc.feedZKFileRegistries(fileRegistries); + } + + private Map<Version, FileRegistry> createFileRegistries() { + FileRegistry a = new MockFileRegistry(); + a.addFile("fileA"); + FileRegistry b = new MockFileRegistry(); + b.addFile("fileB"); + Map<Version, FileRegistry> registryMap = new HashMap<>(); + registryMap.put(Version.fromIntValues(1, 2, 3), a); + registryMap.put(Version.fromIntValues(3, 2, 1), b); + return registryMap; + } + + @Test + public void testInitZooKeeper() throws IOException { + ConfigCurator zk = ConfigCurator.create(new MockCurator()); + BaseDeployLogger logger = new BaseDeployLogger(); + long generation = 1L; + ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, Path.fromString("/1")); + zooKeeperClient.setupZooKeeper(); + String appPath = "/"; + assertThat(zk.getChildren(appPath).size(), is(1)); + assertTrue(zk.exists("/" + String.valueOf(generation))); + String currentAppPath = appPath + String.valueOf(generation); + assertTrue(zk.exists(currentAppPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); + assertThat(zk.getChildren(currentAppPath).size(), is(4)); + } + + @Test + public void testFeedDefFilesToZooKeeper() { + String defsPath = appPath + ConfigCurator.DEFCONFIGS_ZK_SUBPATH; + assertTrue(zk.exists(appPath, ConfigCurator.DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); + List<String> children = zk.getChildren(defsPath); + assertEquals(defsPath + " children", 2, children.size()); + Collections.sort(children); + assertThat(children.get(0), is("a.b.test2,")); + + assertTrue(zk.exists(appPath, ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH.replaceFirst("/", ""))); + String userDefsPath = appPath + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH; + children = zk.getChildren(userDefsPath); + assertThat(children.size(), is(2)); + Collections.sort(children); + assertThat(children.get(0), is("a.b.test2,")); + } + + // TODO: Evaluate if we want this or not + @Test + @Ignore + public void testFeedComponentsFileReferencesToZooKeeper() throws IOException { + final String appDir = "src/test/apps/app_sdbundles"; + ConfigCurator zk = ConfigCurator.create(new MockCurator()); + BaseDeployLogger logger = new BaseDeployLogger(); + Path app = Path.fromString("/1"); + ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); + zooKeeperClient.setupZooKeeper(); + + String currentAppPath = app.getAbsolute(); + assertTrue(zk.exists(currentAppPath, ConfigCurator.USERAPP_ZK_SUBPATH.replaceFirst("/", ""))); + assertTrue(zk.exists(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH, "components")); + assertTrue(zk.exists(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/components", "testbundle.jar")); + assertTrue(zk.exists(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/components", "testbundle2.jar")); + String data = zk.getData(currentAppPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/components", "testbundle2.jar"); + assertThat(data, is(new File(appDir + "/components/testbundle2.jar").getAbsolutePath())); + } + + @Test + public void testFeedUserDefinedFiles() { + assertEquals(zk.getData(appPath+ ConfigCurator.USERAPP_ZK_SUBPATH + "/files", "foo.json"), "foo : foo\n"); + assertEquals(zk.getData(appPath+ ConfigCurator.USERAPP_ZK_SUBPATH + "/files/sub", "bar.json"), "bar : bar\n"); + } + + @Test + public void testFeedAppMetaDataToZooKeeper() { + assertTrue(zk.exists(appPath, ConfigCurator.META_ZK_PATH)); + ApplicationMetaData metaData = ApplicationMetaData.fromJsonString(zk.getData(appPath, ConfigCurator.META_ZK_PATH)); + assertThat(metaData.getApplicationName(), is("appName")); + assertTrue(metaData.getCheckSum().length() > 0); + assertThat(metaData.getDeployedByUser(), is("foo")); + assertThat(metaData.getDeployPath(), is("/bar/baz")); + assertThat(metaData.getDeployTimestamp(), is(1345l)); + assertThat(metaData.getGeneration(), is(3l)); + assertThat(metaData.getPreviousActiveGeneration(), is(2l)); + } + + @Test + public void testVersionedFileRegistry() { + String fileRegPath = appPath + "/" + ZKApplicationPackage.fileRegistryNode; + assertTrue(zk.exists(fileRegPath)); + assertTrue(zk.exists(fileRegPath + "/1.2.3")); + assertTrue(zk.exists(fileRegPath + "/3.2.1")); + // assertNull("Data at " + fileRegPath, zk.getData(fileRegPath)); Not null any more .. hm + } + + @Test + public void include_dirs_are_written_to_ZK() { + assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/" + "dir1", "default.xml")); + assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/nested/" + "dir2", "chain2.xml")); + assertTrue(zk.exists(appPath + ConfigCurator.USERAPP_ZK_SUBPATH + "/nested/" + "dir2", "chain3.xml")); + } + + @Test + public void search_chain_dir_written_to_ZK() { + assertTrue(zk.exists(appPath().append("search").append("chains").append("dir1").append("default.xml").getAbsolute())); + assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain2.xml").getAbsolute())); + assertTrue(zk.exists(appPath().append("search").append("chains").append("dir2").append("chain3.xml").getAbsolute())); + } + + @Test + public void search_definitions_written_to_ZK() { + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("music.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("base.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("video.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("book.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("pc.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("laptop.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("product.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("sock.sd").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("foo.expression").getAbsolute())); + assertTrue(zk.exists(appPath().append(ApplicationPackage.SEARCH_DEFINITIONS_DIR).append("bar.expression").getAbsolute())); + } + + private Path appPath() { + return Path.fromString(appPath).append(ConfigCurator.USERAPP_ZK_SUBPATH); + } + + @Test + public void testWritingHostNamesToZooKeeper() throws IOException { + ConfigCurator zk = ConfigCurator.create(new MockCurator()); + BaseDeployLogger logger = new BaseDeployLogger(); + Path app = Path.fromString("/1"); + ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); + zooKeeperClient.setupZooKeeper(); + zooKeeperClient.feedProvisionInfos(createProvisionInfos()); + Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); + assertTrue(zk.exists(hostsPath.getAbsolute())); + assertEquals(0, zk.getBytes(hostsPath.getAbsolute()).length); // Changed from null + assertTrue(zk.exists(hostsPath.append("1.2.3").getAbsolute())); + assertTrue(zk.exists(hostsPath.append("3.2.1").getAbsolute())); + assertTrue(zk.getBytes(hostsPath.append("1.2.3").getAbsolute()).length > 0); + assertTrue(zk.getBytes(hostsPath.append("3.2.1").getAbsolute()).length > 0); + } + + private Map<Version, ProvisionInfo> createProvisionInfos() { + Map<Version, ProvisionInfo> provisionInfoMap = new HashMap<>(); + ProvisionInfo a = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("host.yahoo.com", Collections.emptyList()))); + ProvisionInfo b = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("host2.yahoo.com", Collections.emptyList()))); + provisionInfoMap.put(Version.fromIntValues(1, 2, 3), a); + provisionInfoMap.put(Version.fromIntValues(3, 2, 1), b); + return provisionInfoMap; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java new file mode 100644 index 00000000000..f69e3aa87fa --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.deploy; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.prelude.semantics.parser.ParseException; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.logging.Level; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author lulf + * @since 5.1 + */ +public class ZooKeeperDeployerTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private static final String defFile = "test2.def"; + + @Test + public void require_that_deployer_is_initialized() throws IOException, ParseException { + ConfigCurator zkfacade = ConfigCurator.create(new MockCurator()); + File serverdbDir = folder.newFolder("serverdb"); + File defsDir = new File(serverdbDir, "serverdefs"); + try { + IOUtils.createWriter(new File(defsDir, defFile), true); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), zkfacade, Path.fromString("/1")); + deploy(FilesApplicationPackage.fromFile(new File("src/test/apps/content")), zkfacade, Path.fromString("/2")); + } + + public void deploy(ApplicationPackage applicationPackage, ConfigCurator configCurator, Path appPath) throws IOException { + MockDeployLogger logger = new MockDeployLogger(); + ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, true, appPath); + ZooKeeperDeployer deployer = new ZooKeeperDeployer(client); + + deployer.deploy(applicationPackage, Collections.singletonMap(Version.fromIntValues(1, 0, 0), new MockFileRegistry()), Collections.emptyMap()); + assertTrue(configCurator.exists(appPath.getAbsolute())); + } + + private static class MockDeployLogger implements DeployLogger { + @Override + public void log(Level level, String message) { } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java new file mode 100644 index 00000000000..02626bffa9d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionLockTest.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.vespa.config.server.TestWithCurator; +import com.yahoo.vespa.curator.recipes.CuratorLockException; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author lulf + */ +public class FileDistributionLockTest extends TestWithCurator { + + FileDistributionLock lock; + private int value = 0; + + @Before + public void setupLock() { + lock = new FileDistributionLock(curator, "/lock"); + value = 0; + } + + @Test + public void testDistributedLock() throws InterruptedException, TimeoutException, ExecutionException { + ExecutorService executor = Executors.newFixedThreadPool(20); + + List<Future<?>> futureList = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + futureList.add(executor.submit(() -> { + lock.lock(); + value++; + lock.unlock(); + })); + } + + for (Future<?> future : futureList) { + future.get(600, TimeUnit.SECONDS); + } + assertThat(value, is(20)); + } + + @Test + public void testDistributedTryLockFailure() throws InterruptedException { + MockCurator mockCurator = new MockCurator(); + lock = new FileDistributionLock(mockCurator, "/mocklock"); + mockCurator.timeoutOnLock = true; + assertFalse(lock.tryLock(600, TimeUnit.SECONDS)); + mockCurator.timeoutOnLock = false; + // Second time should not be blocking + Thread t = new Thread(() -> { + try { + if (lock.tryLock(6, TimeUnit.SECONDS)) { + value = 1; + lock.unlock(); + } + } catch (InterruptedException e) { + } + }); + assertThat(value, is(0)); + t.start(); + t.join(); + assertThat(value, is(1)); + } + + @Test + public void testDistributedLockExceptionFailure() throws InterruptedException { + MockCurator mockCurator = new MockCurator(); + lock = new FileDistributionLock(mockCurator, "/mocklock"); + mockCurator.throwExceptionOnLock = true; + try { + lock.lock(); + fail("Lock call should not succeed"); + } catch (CuratorLockException e) { + // ignore + } + mockCurator.throwExceptionOnLock = false; + // Second time should not be blocking + Thread t = new Thread(() -> { + try { + lock.lock(); + value = 1; + lock.unlock(); + } catch (Exception e) { + fail("Should not fail"); + } + }); + assertThat(value, is(0)); + t.start(); + t.join(); + assertThat(value, is(1)); + } + + @Test(expected = UnsupportedOperationException.class) + public void testConditionNotSupported() { + lock.newCondition(); + } + + @Test(expected = UnsupportedOperationException.class) + public void testLockInterruptiblyNotSupported() throws InterruptedException { + lock.lockInterruptibly(); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java new file mode 100644 index 00000000000..a3ba3b26b50 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/ContentHandlerTestBase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.Response.Status.OK; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import javax.annotation.Nullable; + +import org.junit.Test; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.Collections2; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest; + +public abstract class ContentHandlerTestBase extends SessionHandlerTest { + protected String baseUrl = "http://foo:1337/application/v2/tenant/default/session/1/content/"; + + @Test + public void require_that_content_can_be_retrieved() throws IOException { + assertContent("/test.txt", "foo\n"); + assertContent("/foo/", generateResultArray("foo/bar/", "foo/test1.txt", "foo/test2.txt")); + assertContent("/foo", generateResultArray("foo/")); + assertContent("/foo/test1.txt", "bar\n"); + assertContent("/foo/test2.txt", "baz\n"); + assertContent("/foo/bar/", generateResultArray("foo/bar/test.txt")); + assertContent("/foo/bar", generateResultArray("foo/bar/")); + assertContent("/foo/bar/test.txt", "bim\n"); + assertContent("/foo/?recursive=true", generateResultArray("foo/bar/", "foo/bar/test.txt", "foo/test1.txt", "foo/test2.txt")); + } + + @Test + public void require_that_nonexistant_file_returns_not_found() throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.GET, "/test2.txt"); + assertNotNull(response); + assertThat(response.getStatus(), is(NOT_FOUND)); + } + + @Test + public void require_that_return_property_is_used() throws IOException { + assertContent("/test.txt?return=content", "foo\n"); + } + + @Test + public void require_that_illegal_return_property_fails() { + HttpResponse response = doRequest(HttpRequest.Method.GET, "/test.txt?return=foo"); + assertThat(response.getStatus(), is(BAD_REQUEST)); + } + + @Test + public void require_that_status_can_be_retrieved() throws IOException { + assertStatus("/test.txt?return=status", + "{\"status\":\"new\",\"md5\":\"d3b07384d113edec49eaa6238ad5ff00\",\"name\":\"" + baseUrl + "test.txt\"}"); + assertStatus("/foo/?return=status", + "[{\"status\":\"new\",\"md5\":\"\",\"name\":\"" + baseUrl + "foo/bar\"}," + + "{\"status\":\"new\",\"md5\":\"c157a79031e1c40f85931829bc5fc552\",\"name\":\"" + baseUrl + "foo/test1.txt\"}," + + "{\"status\":\"new\",\"md5\":\"258622b1688250cb619f3c9ccaefb7eb\",\"name\":\"" + baseUrl + "foo/test2.txt\"}]"); + assertStatus("/foo/?return=status&recursive=true", + "[{\"status\":\"new\",\"md5\":\"\",\"name\":\"" + baseUrl + "foo/bar\"}," + + "{\"status\":\"new\",\"md5\":\"579cae6111b269c0129af36a2243b873\",\"name\":\"" + baseUrl + "foo/bar/test.txt\"}," + + "{\"status\":\"new\",\"md5\":\"c157a79031e1c40f85931829bc5fc552\",\"name\":\"" + baseUrl + "foo/test1.txt\"}," + + "{\"status\":\"new\",\"md5\":\"258622b1688250cb619f3c9ccaefb7eb\",\"name\":\"" + baseUrl + "foo/test2.txt\"}]"); + } + + protected void assertContent(String path, String expectedContent) throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.GET, path); + assertNotNull(response); + final String renderedString = SessionHandlerTest.getRenderedString(response); + assertThat(renderedString, response.getStatus(), is(OK)); + assertThat(renderedString, is(expectedContent)); + } + + protected void assertStatus(String path, String expectedContent) throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.GET, path); + assertNotNull(response); + final String renderedString = SessionHandlerTest.getRenderedString(response); + assertThat(renderedString, response.getStatus(), is(OK)); + assertThat(renderedString, is(expectedContent)); + } + + protected abstract HttpResponse doRequest(HttpRequest.Method method, String path); + + private String generateResultArray(String... files) { + Collection<String> output = Collections2.transform(Arrays.asList(files), new Function<String, String>() { + @Override + public String apply(@Nullable String input) { + return "\"" + baseUrl + input + "\""; + } + }); + StringBuilder sb = new StringBuilder(); + sb.append("["); + sb.append(Joiner.on(",").join(output)); + sb.append("]"); + return sb.toString(); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java new file mode 100644 index 00000000000..22607e8fc26 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HandlerTest.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * Base class for handler tests + * + * @author musum + * @since 5.1.14 + */ +public class HandlerTest { + public static void assertHttpStatusCodeErrorCodeAndMessage(HttpResponse response, int statusCode, HttpErrorResponse.errorCodes errorCode, String message) throws IOException { + assertNotNull(response); + String renderedString = SessionHandlerTest.getRenderedString(response); + if (renderedString == null) { + renderedString = "assert failed"; + } + assertThat(renderedString, response.getStatus(), is(statusCode)); + if (errorCode != null) { + assertThat(renderedString, containsString(errorCode.name())); + } + assertThat(renderedString, containsString(message)); + } + + public static void assertHttpStatusCodeAndMessage(HttpResponse response, int statusCode, String message) throws IOException { + assertHttpStatusCodeErrorCodeAndMessage(response, statusCode, null, message); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigRequestTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigRequestTest.java new file mode 100644 index 00000000000..8525a152403 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigRequestTest.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import java.io.IOException; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.vespa.config.ConfigKey; + +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class HttpConfigRequestTest { + @Test + public void require_that_request_can_be_created() { + final ConfigKey<?> configKey = new ConfigKey<>("foo", "myid", "bar"); + + HttpConfigRequest request = HttpConfigRequest.createFromRequestV1(HttpRequest.createTestRequest("http://example.yahoo.com:8080/config/v1/" + + configKey.getNamespace() + "." + configKey.getName() + "/" + configKey.getConfigId(), GET)); + assertThat(request.getConfigKey(), is(configKey)); + assertTrue(request.getDefContent().isEmpty()); + } + + @Test + public void require_namespace_can_have_dots() { + final ConfigKey<?> configKey = new ConfigKey<>("foo", "myid", "bar.baz"); + HttpConfigRequest request = HttpConfigRequest.createFromRequestV1(HttpRequest.createTestRequest("http://example.yahoo.com:8080/config/v1/" + + configKey.getNamespace() + "." + configKey.getName() + "/" + configKey.getConfigId(), GET)); + assertEquals(request.getConfigKey().getNamespace(), "bar.baz"); + } + + @Test + public void require_that_request_can_be_created_with_advanced_uri() throws IOException { + HttpConfigRequest.createFromRequestV1(HttpRequest.createTestRequest( + "http://example.yahoo.com:19071/config/v1/vespa.config.cloud.sentinel/host-01.example.yahoo.com", GET)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java new file mode 100644 index 00000000000..b7d41e2835e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.protocol.ConfigResponse; + +import com.yahoo.vespa.config.protocol.SlimeConfigResponse; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringReader; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class HttpConfigResponseTest { + @Test + public void require_that_response_is_created_from_config() throws IOException { + final long generation = 1L; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + // TODO: Hope to be able to remove this mess soon. + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + InnerCNode targetDef = dParser.getTree(); + ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5"); + HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpErrorResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpErrorResponseTest.java new file mode 100644 index 00000000000..cd18823ea1c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpErrorResponseTest.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +import static com.yahoo.jdisc.http.HttpResponse.Status.*; + +/** + * @author lulf + * @since 5.1 + */ +public class HttpErrorResponseTest { + @Test + public void testThatHttpErrorResponseIsRenderedAsJson() throws IOException { + HttpErrorResponse response = HttpErrorResponse.badRequest("Error doing something"); + assertThat(response.getJdiscResponse().getStatus(), is(BAD_REQUEST)); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"error-code\":\"BAD_REQUEST\",\"message\":\"Error doing something\"}")); + } + + @Test + public void testThatHttpErrorResponseProvidesCorrectErrorMessage() throws IOException { + HttpErrorResponse response = HttpErrorResponse.badRequest("Error doing something"); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Error doing something"); + } + + @Test + public void testThatHttpErrorResponseHasJsonContentType() throws IOException { + HttpErrorResponse response = HttpErrorResponse.badRequest("Error doing something"); + assertThat(response.getContentType(), is("application/json")); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java new file mode 100644 index 00000000000..d1c4c5e5e2e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.server.MockRequestHandler; +import com.yahoo.vespa.config.protocol.SlimeConfigResponse; +import com.yahoo.config.provision.ApplicationId; + +import org.junit.Before; +import org.junit.Test; +import java.io.IOException; +import java.io.StringReader; +import java.util.Collections; +import java.util.HashSet; +import java.util.concurrent.Executor; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.jdisc.http.HttpResponse.Status.*; + + +/** + * @author lulf + * @since 5.1 + */ +public class HttpGetConfigHandlerTest { + private static final String configUri = "http://yahoo.com:8080/config/v1/foo.bar/myid"; + + private MockRequestHandler mockRequestHandler; + private HttpGetConfigHandler handler; + + @Before + public void setUp() { + mockRequestHandler = new MockRequestHandler(); + mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>() {{ + add(new ConfigKey<>("bar", "myid", "foo")); + }} ); + handler = new HttpGetConfigHandler(new Executor() { + @SuppressWarnings("NullableProblems") + @Override + public void execute(Runnable command) { + command.run(); + } + }, mockRequestHandler, AccessLog.voidAccessLog()); + } + + @Test + public void require_that_handler_can_be_created() throws IOException { + // Define config response for mock handler + final long generation = 1L; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + InnerCNode targetDef = getInnerCNode(); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); + } + + @Test + public void require_correct_error_response() throws IOException { + final String nonExistingConfigNameUri = "http://yahoo.com:8080/config/v1/nonexisting.config/myid"; + final String nonExistingConfigUri = "http://yahoo.com:8080/config/v1/foo.bar/myid/nonexisting/id"; + final String illegalConfigNameUri = "http://yahoo.com:8080/config/v1/foobar/myid"; + + HttpResponse response = handler.handle(HttpRequest.createTestRequest(nonExistingConfigNameUri, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config: nonexisting.config"); + assertTrue(SessionHandlerTest.getRenderedString(response).contains("No such config:")); + response = handler.handle(HttpRequest.createTestRequest(nonExistingConfigUri, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config id: myid/nonexisting/id"); + assertEquals(response.getContentType(), "application/json"); + assertTrue(SessionHandlerTest.getRenderedString(response).contains("No such config id:")); + response = handler.handle(HttpRequest.createTestRequest(illegalConfigNameUri, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Illegal config, must be of form namespace.name."); + } + + @Test + public void require_that_nocache_property_works() throws IOException { + long generation = 1L; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + InnerCNode targetDef = getInnerCNode(); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); + HttpResponse response = handler.handle(request); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); + } + + private InnerCNode getInnerCNode() { + // TODO: Hope to be able to remove this mess soon. + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + return dParser.getTree(); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java new file mode 100644 index 00000000000..7868909f65f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpHandlerTest.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.34 + */ +public class HttpHandlerTest { + @Test + public void testResponse() throws IOException { + final String message = "failed"; + HttpHandler httpHandler = new HttpTestHandler(Executors.newSingleThreadExecutor(), AccessLog.voidAccessLog(), new InvalidApplicationException(message)); + HttpResponse response = httpHandler.handle(HttpRequest.createTestRequest("foo", com.yahoo.jdisc.http.HttpRequest.Method.GET)); + assertThat(response.getStatus(), is(Response.Status.BAD_REQUEST)); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + Slime data = new Slime(); + new JsonDecoder().decode(data, baos.toByteArray()); + assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.INVALID_APPLICATION_PACKAGE.name())); + assertThat(data.get().field("message").asString(), is(message)); + } + + private static class HttpTestHandler extends HttpHandler { + private RuntimeException exception; + public HttpTestHandler(Executor executor, AccessLog accessLog, RuntimeException exception) { + super(executor, accessLog); + this.exception = exception; + } + + @Override + public HttpResponse handleGET(HttpRequest request) { + throw exception; + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java new file mode 100644 index 00000000000..ad917c5db6d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.MockRequestHandler; +import com.yahoo.vespa.config.server.http.HttpListConfigsHandler.ListConfigsResponse; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.Executor; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; + +/** + * @author lulf + * @since 5.1 + */ +public class HttpListConfigsHandlerTest { + + private MockRequestHandler mockRequestHandler; + private HttpListConfigsHandler handler; + private HttpListNamedConfigsHandler namedHandler; + + @Before + public void setUp() { + mockRequestHandler = new MockRequestHandler(); + mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>() {{ + add(new ConfigKey<>("bar", "conf/id/", "foo")); + }} ); + handler = new HttpListConfigsHandler(new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }, AccessLog.voidAccessLog(), mockRequestHandler); + namedHandler = new HttpListNamedConfigsHandler(new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }, mockRequestHandler, AccessLog.voidAccessLog()); + } + + @Test + public void require_that_handler_can_be_created() throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest("/config/v1/", GET)); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"children\":[],\"configs\":[]}")); + } + + @Test + public void require_that_named_handler_can_be_created() throws IOException { + HttpRequest req = HttpRequest.createTestRequest("http://foo.com:8080/config/v1/foo.bar/conf/id/", GET); + req.getJDiscRequest().parameters().put("http.path", Arrays.asList("foo.bar")); + HttpResponse response = namedHandler.handle(req); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"children\":[],\"configs\":[]}")); + } + + @Test + public void require_child_listings_correct() { + Set<ConfigKey<?>> keys = new LinkedHashSet<ConfigKey<?>>() {{ + add(new ConfigKey<>("name1", "id/1", "ns1")); + add(new ConfigKey<>("name1", "id/1", "ns1")); + add(new ConfigKey<>("name1", "id/2", "ns1")); + add(new ConfigKey<>("name1", "", "ns1")); + add(new ConfigKey<>("name1", "id/1/1", "ns1")); + add(new ConfigKey<>("name1", "id2", "ns1")); + add(new ConfigKey<>("name1", "id/2/1", "ns1")); + add(new ConfigKey<>("name1", "id/2/1/5/6", "ns1")); + }}; + Set<ConfigKey<?>> keysThatHaveChild = HttpListConfigsHandler.ListConfigsResponse.keysThatHaveAChildWithSameName(keys, keys); + assertEquals(keysThatHaveChild.size(), 3); + } + + @Test + public void require_url_building_and_mimetype_correct() { + HttpListConfigsHandler.ListConfigsResponse resp = new ListConfigsResponse(new HashSet<ConfigKey<?>>(), null, "http://foo.com/config/v1/", true); + assertEquals(resp.toUrl(new ConfigKey<>("myconfig", "my/id", "mynamespace"), true), "http://foo.com/config/v1/mynamespace.myconfig/my/id"); + assertEquals(resp.toUrl(new ConfigKey<>("myconfig", "my/id", "mynamespace"), false), "http://foo.com/config/v1/mynamespace.myconfig/my/id/"); + assertEquals(resp.getContentType(), "application/json"); + + } + + @Test + public void require_error_on_bad_request() throws IOException { + HttpRequest req = HttpRequest.createTestRequest("http://foo.com:8080/config/v1/foobar/conf/id/", GET); + HttpResponse resp = namedHandler.handle(req); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(resp, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Illegal config, must be of form namespace.name."); + req = HttpRequest.createTestRequest("http://foo.com:8080/config/v1/foo.barNOPE/conf/id/", GET); + resp = namedHandler.handle(req); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(resp, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config: foo.barNOPE"); + req = HttpRequest.createTestRequest("http://foo.com:8080/config/v1/foo.bar/conf/id/NOPE/", GET); + resp = namedHandler.handle(req); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(resp, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config id: conf/id/NOPE/"); + } + + @Test + public void require_correct_error_response_on_no_model() throws IOException { + mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>()); + HttpResponse response = namedHandler.handle(HttpRequest.createTestRequest("http://yahoo.com:8080/config/v1/foo.bar/myid/", GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, + HttpErrorResponse.errorCodes.NOT_FOUND, + "Config not available, verify that an application package has been deployed and activated."); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerTestBase.java new file mode 100644 index 00000000000..2e5576869b5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerTestBase.java @@ -0,0 +1,266 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import static com.yahoo.jdisc.Response.Status.*; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; + +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.provision.*; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.SuperModelGenerationCounter; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; +import com.yahoo.vespa.config.server.session.*; + +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModelFactory; +import org.hamcrest.core.Is; +import org.junit.Test; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.DeployData; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.vespa.config.server.HostRegistry; +import com.yahoo.vespa.config.server.PathProvider; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +public abstract class SessionActiveHandlerTestBase extends SessionHandlerTest { + + private File testApp = new File("src/test/apps/app"); + protected static final String appName = "default"; + protected TenantName tenant = null; + protected ConfigCurator configCurator; + protected Curator curator; + protected RemoteSessionRepo remoteSessionRepo; + protected LocalSessionRepo localRepo; + protected PathProvider pathProvider; + protected ApplicationRepo applicationRepo; + protected String activatedMessage = " activated."; + protected String tenantMessage = ""; + + @Test + public void testThatPreviousSessionIsDeactivated() throws Exception { + RemoteSession firstSession = activateAndAssertOK(90l, 0l); + activateAndAssertOK(91l, 90l); + assertThat(firstSession.getStatus(), is(Session.Status.DEACTIVATE)); + } + + @Test + public void testForceActivationWithActivationInBetween() throws Exception { + activateAndAssertOK(90l, 0l); + activateAndAssertOK(92l, 89l, "?force=true"); + } + + @Test + public void testUnknownSession() throws Exception { + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, 9999L, "?timeout=1.0")); + assertEquals(response.getStatus(), 404); + } + + @Test + public void testActivationWithActivationInBetween() throws Exception { + activateAndAssertOK(90l, 0l); + activateAndAssertError(92l, 89l, + HttpErrorResponse.errorCodes.BAD_REQUEST, + getActivateLogPre() + + "Cannot activate session 92 because the currently active session (90) has changed since session 92 was created (was 89 at creation time)"); + } + + protected abstract String getActivateLogPre(); + + @Test + public void testActivationOfUnpreparedSession() throws Exception { + // Needed so we can test that previous active session is still active after a failed activation + RemoteSession firstSession = activateAndAssertOK(90l, 0l); + long sessionId = 91l; + ActivateRequest activateRequest = new ActivateRequest(sessionId, 0l, Session.Status.NEW, "").invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + RemoteSession session = activateRequest.getSession(); + assertThat(actResponse.getStatus(), is(BAD_REQUEST)); + assertThat(getRenderedString(actResponse), is("{\"error-code\":\"BAD_REQUEST\",\"message\":\"" + getActivateLogPre() + "Session " + sessionId + " is not prepared\"}")); + assertThat(session.getStatus(), is(not(Session.Status.ACTIVATE))); + assertThat(firstSession.getStatus(), is(Session.Status.ACTIVATE)); + } + + @Test + public void testActivationWithBarrierTimeout() throws Exception { + // Needed so we can test that previous active session is still active after a failed activation + activateAndAssertOK(90l, 0l); + ((MockCurator) curator).timeoutBarrierOnEnter(true); + ActivateRequest activateRequest = new ActivateRequest(91l, 90l, "").invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + assertThat(actResponse.getStatus(), is(INTERNAL_SERVER_ERROR)); + } + + @Test + public void testActivationOfSessionThatDoesNotExistAsLocalSession() throws Exception { + ActivateRequest activateRequest = new ActivateRequest(90l, 0l, "").invoke(false); + HttpResponse actResponse = activateRequest.getActResponse(); + assertThat(actResponse.getStatus(), is(NOT_FOUND)); + String message = getRenderedString(actResponse); + assertThat(message, is("{\"error-code\":\"NOT_FOUND\",\"message\":\"Session 90 was not found\"}")); + } + + @Test + public void require_that_session_created_from_active_that_is_no_longer_active_cannot_be_activated() throws Exception { + long sessionId = 1; + activateAndAssertOK(1, 0); + sessionId++; + activateAndAssertOK(sessionId, 1); + + sessionId++; + ActivateRequest activateRequest = new ActivateRequest(sessionId, 1, "").invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + String message = getRenderedString(actResponse); + assertThat(message, actResponse.getStatus(), Is.is(BAD_REQUEST)); + assertThat(message, + containsString("Cannot activate session 3 because the currently active session (2) has changed since session 3 was created (was 1 at creation time)")); + } + + @Test + public void testAlreadyActivatedSession() throws Exception { + activateAndAssertOK(1, 0); + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, 1l)); + String message = getRenderedString(response); + assertThat(message, response.getStatus(), Is.is(BAD_REQUEST)); + assertThat(message, containsString("Session 1 is already active")); + } + + protected abstract SessionHandler createHandler() throws Exception; + + private RemoteSession createRemoteSession(long sessionId, Session.Status status, SessionZooKeeperClient zkClient) throws IOException { + zkClient.writeStatus(status); + ZooKeeperClient zkC = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, pathProvider.getSessionDirs().append(String.valueOf(sessionId))); + VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); + zkC.feedZKFileRegistries(Collections.singletonMap(modelFactory.getVersion(), new MockFileRegistry())); + zkC.feedProvisionInfos(Collections.singletonMap(modelFactory.getVersion(), ProvisionInfo.withHosts(Collections.emptySet()))); + RemoteSession session = new RemoteSession(TenantName.from("default"), sessionId, new TestComponentRegistry(curator, configCurator, new ModelFactoryRegistry(Collections.singletonList(modelFactory))), zkClient); + remoteSessionRepo.addSession(session); + return session; + } + + private LocalSessionRepo addLocalSession(long sessionId, DeployData deployData, SessionZooKeeperClient zkc) { + writeApplicationId(zkc, deployData.getApplicationName()); + TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenant); + ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(testApp, + deployData + ); + localRepo.addSession(new LocalSession(tenant, sessionId, new SessionTest.MockSessionPreparer(), new SessionContext(app, zkc, new File(tenantFileSystemDirs.path(), String.valueOf(sessionId)), applicationRepo, new HostRegistry<>(), new SuperModelGenerationCounter(curator)))); + return localRepo; + } + + protected abstract void writeApplicationId(SessionZooKeeperClient zkc, String applicationName); + + protected abstract Session activateAndAssertOK(long sessionId, long previousSessionId, String subPath) throws Exception; + + protected abstract RemoteSession activateAndAssertOK(long sessionId, long previousSessionId) throws Exception; + + protected ActivateRequest activateAndAssertOKPut(long sessionId, long previousSessionId, String subPath) throws Exception { + ActivateRequest activateRequest = new ActivateRequest(sessionId, previousSessionId, subPath).invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + String message = getRenderedString(actResponse); + assertThat(message, actResponse.getStatus(), is(OK)); + assertActivationMessageOK(activateRequest, message); + RemoteSession session = activateRequest.getSession(); + assertThat(session.getStatus(), is(Session.Status.ACTIVATE)); + return activateRequest; + } + + protected abstract void assertActivationMessageOK(ActivateRequest activateRequest, String message) throws IOException; + + protected abstract void activateAndAssertError(long sessionId, long previousSessionId, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception; + + protected ActivateRequest activateAndAssertErrorPut(long sessionId, long previousSessionId, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { + ActivateRequest activateRequest = new ActivateRequest(sessionId, previousSessionId, "").invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + RemoteSession session = activateRequest.getSession(); + assertThat(actResponse.getStatus(), is(BAD_REQUEST)); + String message = getRenderedString(actResponse); + assertThat(message, is("{\"error-code\":\"" + errorCode.name() + "\",\"message\":\"" + expectedError + "\"}")); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + return activateRequest; + } + + protected void testUnsupportedMethod(com.yahoo.container.jdisc.HttpRequest request) throws Exception { + HttpResponse response = createHandler().handle(request); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, METHOD_NOT_ALLOWED, + HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, + "Method '" + request.getMethod().name() + "' is not supported"); + } + + protected class ActivateRequest { + private long sessionId; + private RemoteSession session; + private SessionHandler handler; + private HttpResponse actResponse; + private Session.Status initialStatus; + private DeployData deployData; + private ApplicationMetaData metaData; + private String subPath; + + public ActivateRequest(long sessionId, long previousSessionId, String subPath) { + this(sessionId, previousSessionId, Session.Status.PREPARE, subPath); + } + + public ActivateRequest(long sessionId, long previousSessionId, Session.Status initialStatus, String subPath) { + this.sessionId = sessionId; + this.initialStatus = initialStatus; + this.deployData = new DeployData("foo", "bar", appName, 0l, sessionId, previousSessionId); + this.subPath = subPath; + } + + public RemoteSession getSession() { + return session; + } + + public SessionHandler getHandler() { + return handler; + } + + public HttpResponse getActResponse() { + return actResponse; + } + + public long getSessionId() { + return sessionId; + } + + public ApplicationMetaData getMetaData() { + return metaData; + } + + public ActivateRequest invoke() throws Exception { + return invoke(true); + } + + public ActivateRequest invoke(boolean createLocalSession) throws Exception { + SessionZooKeeperClient zkClient = new MockSessionZKClient(curator, pathProvider.getSessionDirs().append(String.valueOf(sessionId)), Optional.of(ProvisionInfo.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); + session = createRemoteSession(sessionId, initialStatus, zkClient); + if (createLocalSession) { + LocalSessionRepo repo = addLocalSession(sessionId, deployData, zkClient); + metaData = repo.getSession(sessionId).getMetaData(); + } + handler = createHandler(); + actResponse = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); + return this; + } + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionContentHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionContentHandlerTestBase.java new file mode 100644 index 00000000000..43fdb2d747a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionContentHandlerTestBase.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.google.common.io.Files; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.text.Utf8; +import org.apache.commons.io.FileUtils; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +public abstract class SessionContentHandlerTestBase extends ContentHandlerTestBase { + + @Test + public void require_that_directories_can_be_created() throws IOException { + assertMkdir("/bar/"); + assertMkdir("/bar/brask/"); + assertMkdir("/bar/brask/"); + assertMkdir("/bar/brask/bram/"); + assertMkdir("/brask/og/bram/"); + }// TODO: Enable when we have a predictable way of checking request body existence. + + @Test + @Ignore + public void require_that_mkdir_with_body_is_illegal() throws IOException { + HttpResponse response = put("/foobio/", "foo"); + assertNotNull(response); + assertThat(response.getStatus(), is(Response.Status.BAD_REQUEST)); + } + + @Test + public void require_that_nonexistant_session_returns_not_found() throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.GET, "/test.txt", 2l); + assertNotNull(response); + assertThat(response.getStatus(), is(Response.Status.NOT_FOUND)); + } + + protected HttpResponse put(String path, String content) { + ByteArrayInputStream data = new ByteArrayInputStream(Utf8.toBytes(content)); + return doRequest(HttpRequest.Method.PUT, path, data); + } + + @Test + public void require_that_file_write_without_body_is_illegal() throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.PUT, "/foobio.txt"); + assertNotNull(response); + assertThat(response.getStatus(), is(Response.Status.BAD_REQUEST)); + } + + @Test + public void require_that_files_can_be_written() throws IOException { + assertWriteFile("/foo/minfil.txt", "Mycontent"); + assertWriteFile("/foo/minfil.txt", "Differentcontent"); + } + + @Test + public void require_that_nonexistant_file_returs_not_found_when_deleted() throws IOException { + assertDeleteFile(Response.Status.NOT_FOUND, "/test2.txt", "{\"error-code\":\"NOT_FOUND\",\"message\":\"Session 1 does not contain a file 'test2.txt'\"}"); + } + + @Test + public void require_that_files_can_be_deleted() throws IOException { + assertDeleteFile(Response.Status.OK, "/test.txt"); + assertDeleteFile(Response.Status.NOT_FOUND, "/test.txt", "{\"error-code\":\"NOT_FOUND\",\"message\":\"Session 1 does not contain a file 'test.txt'\"}"); + assertDeleteFile(Response.Status.BAD_REQUEST, "/newtest", "{\"error-code\":\"BAD_REQUEST\",\"message\":\"File 'newtest' is not an empty directory\"}"); + assertDeleteFile(Response.Status.OK, "/newtest/testfile.txt"); + assertDeleteFile(Response.Status.OK, "/newtest"); + } + + @Test + public void require_that_status_is_given_for_new_files() throws IOException { + assertStatus("/test.txt?return=status", + "{\"status\":\"new\",\"md5\":\"d3b07384d113edec49eaa6238ad5ff00\",\"name\":\"http://foo:1337" + pathPrefix + "1/content/test.txt\"}"); + assertWriteFile("/test.txt", "Mycontent"); + assertStatus("/test.txt?return=status", + "{\"status\":\"changed\",\"md5\":\"01eabd73c69d78d0009ec93cd62d7f77\",\"name\":\"http://foo:1337" + pathPrefix + "1/content/test.txt\"}"); + } + + private void assertWriteFile(String path, String content) throws IOException { + HttpResponse response = put(path, content); + assertNotNull(response); + assertThat(response.getStatus(), is(Response.Status.OK)); + assertContent(path, content); + assertThat(SessionHandlerTest.getRenderedString(response), + is("{\"prepared\":\"http://foo:1337" + pathPrefix + "1/prepared\"}")); + } + + private void assertDeleteFile(int statusCode, String filePath) throws IOException { + assertDeleteFile(statusCode, filePath, "{\"prepared\":\"http://foo:1337" + pathPrefix + "1/prepared\"}"); + } + + private void assertDeleteFile(int statusCode, String filePath, String expectedResponse) throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.DELETE, filePath); + assertNotNull(response); + assertThat(response.getStatus(), is(statusCode)); + assertThat(SessionHandlerTest.getRenderedString(response), is(expectedResponse)); + } + + private void assertMkdir(String path) throws IOException { + HttpResponse response = doRequest(HttpRequest.Method.PUT, path); + assertNotNull(response); + assertThat(response.getStatus(), is(Response.Status.OK)); + assertThat(SessionHandlerTest.getRenderedString(response), + is("{\"prepared\":\"http://foo:1337" + pathPrefix + "1/prepared\"}")); + } + + protected File createTestApp() throws IOException { + File testApp = Files.createTempDir(); + FileUtils.copyDirectory(new File("src/test/apps/content"), testApp); + return testApp; + } + + protected abstract HttpResponse doRequest(HttpRequest.Method method, String path, long sessionId); + protected abstract HttpResponse doRequest(HttpRequest.Method method, String path, InputStream data); + protected abstract HttpResponse doRequest(HttpRequest.Method method, String path, long sessionId, InputStream data); +}
\ No newline at end of file diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionCreateHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionCreateHandlerTestBase.java new file mode 100644 index 00000000000..6a6a4097319 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionCreateHandlerTestBase.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.google.common.io.Files; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.server.CompressedApplicationInputStreamTest; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.SessionFactory; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.jdisc.Response.Status.*; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * Tests for session create handlers, to make it easier to have + * similar tests for more than one version of the API. + * + * @author musum + * @since 5.1.28 + */ +public abstract class SessionCreateHandlerTestBase extends SessionHandlerTest { + + public static final HashMap<String, String> postHeaders = new HashMap<>(); + + protected String pathPrefix = "/application/v2/session/"; + protected String createdMessage = " created.\""; + protected String tenantMessage = ""; + + public File testApp = new File("src/test/apps/app"); + public LocalSessionRepo localSessionRepo; + public ApplicationRepo applicationRepo; + + static { + postHeaders.put(SessionCreate.contentTypeHeader, SessionCreate.APPLICATION_X_GZIP); + } + + @Ignore + @Test + public void require_that_from_parameter_cannot_be_set_if_data_in_request() throws IOException { + HttpRequest request = post(Collections.singletonMap("from", "active")); + HttpResponse response = createHandler().handle(request); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Parameter 'from' is illegal for POST"); + } + + @Test + public void require_that_post_request_must_contain_data() throws IOException { + HttpResponse response = createHandler().handle(post()); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no data"); + } + + @Test + public void require_that_post_request_must_have_correct_content_type() throws IOException { + HashMap<String, String> headers = new HashMap<>(); // no Content-Type header + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + HttpResponse response = createHandler().handle(post(outFile, headers, null)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Request contains no Content-Type header"); + } + + @Test + public void require_that_application_name_is_given_from_parameter() throws IOException { + Map<String, String> params = Collections.singletonMap("name", "ulfio"); + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + MockSessionFactory factory = new MockSessionFactory(); + createHandler(factory).handle(post(outFile, postHeaders, params)); + assertTrue(factory.createCalled); + assertThat(factory.applicationName, is("ulfio")); + } + + protected void assertFromParameter(String expected, String from) throws IOException { + HttpRequest request = post(Collections.singletonMap("from", from)); + MockSessionFactory factory = new MockSessionFactory(); + factory.applicationPackage = testApp; + HttpResponse response = createHandler(factory).handle(request); + assertNotNull(response); + assertThat(response.getStatus(), is(OK)); + assertTrue(factory.createFromCalled); + assertThat(SessionHandlerTest.getRenderedString(response), + is("{\"log\":[]" + tenantMessage + ",\"session-id\":\"" + expected + "\",\"prepared\":\"http://" + hostname + ":" + port + pathPrefix + + expected + "/prepared\",\"content\":\"http://" + hostname + ":" + port + pathPrefix + + expected + "/content/\",\"message\":\"Session " + expected + createdMessage + "}")); + } + + protected void assertIllegalFromParameter(String fromValue) throws IOException { + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + HttpRequest request = post(outFile, postHeaders, Collections.singletonMap("from", fromValue)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(createHandler().handle(request), BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Parameter 'from' has illegal value '" + fromValue + "'"); + } + + @Test + public void require_that_prepare_url_is_returned_on_success() throws IOException { + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + Map<String, String> parameters = Collections.singletonMap("name", "foo"); + HttpResponse response = createHandler().handle(post(outFile, postHeaders, parameters)); + assertNotNull(response); + assertThat(response.getStatus(), is(OK)); + assertThat(SessionHandlerTest.getRenderedString(response), + is("{\"log\":[]" + tenantMessage + ",\"session-id\":\"0\",\"prepared\":\"http://" + + hostname + ":" + port + pathPrefix + "0/prepared\",\"content\":\"http://" + + hostname + ":" + port + pathPrefix + "0/content/\",\"message\":\"Session 0" + createdMessage + "}")); + } + + @Test + public void require_that_session_factory_is_called() throws IOException { + MockSessionFactory sessionFactory = new MockSessionFactory(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + createHandler(sessionFactory).handle(post(outFile)); + assertTrue(sessionFactory.createCalled); + } + + @Test + public void require_that_handler_does_not_support_get() throws IOException { + HttpResponse response = createHandler().handle(HttpRequest.createTestRequest(pathPrefix, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, METHOD_NOT_ALLOWED, + HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, + "Method 'GET' is not supported"); + } + + @Test + public void require_internal_error_when_exception() throws IOException { + MockSessionFactory factory = new MockSessionFactory(); + factory.doThrow = true; + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + HttpResponse response = createHandler(factory).handle(post(outFile)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, INTERNAL_SERVER_ERROR, + HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR, + "foo"); + } + + @Test + public void require_that_handler_unpacks_application() throws IOException { + MockSessionFactory sessionFactory = new MockSessionFactory(); + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + createHandler(sessionFactory).handle(post(outFile)); + assertTrue(sessionFactory.createCalled); + final File applicationPackage = sessionFactory.applicationPackage; + assertNotNull(applicationPackage); + assertTrue(applicationPackage.exists()); + final File[] files = applicationPackage.listFiles(); + assertNotNull(files); + assertThat(files.length, is(2)); + } + + @Test + public void require_that_session_is_stored_in_repo() throws IOException { + File outFile = CompressedApplicationInputStreamTest.createTarFile(); + createHandler(new MockSessionFactory()).handle(post(outFile)); + assertNotNull(localSessionRepo.getSession(0l)); + } + + public abstract SessionHandler createHandler(); + + public abstract SessionHandler createHandler(SessionFactory sessionFactory); + + public abstract HttpRequest post() throws FileNotFoundException; + + public abstract HttpRequest post(File file) throws FileNotFoundException; + + public abstract HttpRequest post(File file, Map<String, String> headers, Map<String, String> parameters) throws FileNotFoundException; + + public abstract HttpRequest post(Map<String, String> parameters) throws FileNotFoundException; + + public static class MockSessionFactory implements SessionFactory { + public boolean createCalled = false; + public boolean createFromCalled = false; + public boolean doThrow = false; + public File applicationPackage; + public String applicationName; + + @Override + public LocalSession createSession(File applicationDirectory, String applicationName, DeployLogger logger, TimeoutBudget timeoutBudget) { + createCalled = true; + this.applicationName = applicationName; + if (doThrow) { + throw new RuntimeException("foo"); + } + final File tempDir = Files.createTempDir(); + try { + IOUtils.copyDirectory(applicationDirectory, tempDir); + } catch (IOException e) { + e.printStackTrace(); + } + this.applicationPackage = tempDir; + return new SessionHandlerTest.MockSession(0, FilesApplicationPackage.fromFile(applicationPackage)); + } + + @Override + public LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, TimeoutBudget timeoutBudget) { + if (doThrow) { + throw new RuntimeException("foo"); + } + createFromCalled = true; + return new SessionHandlerTest.MockSession(existingSession.getSessionId() + 1, FilesApplicationPackage.fromFile(applicationPackage)); + } + } +} + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java new file mode 100644 index 00000000000..2d7e293fb3c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import org.junit.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpRequest.Method.*; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author musum + * @since 5.1.14 + */ +public class SessionExampleHandlerTest { + private static final String URI = "http://localhost:19071/session/example"; + + @Test + public void basicPut() throws IOException { + final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); + final HttpRequest request = HttpRequest.createTestRequest(URI, PUT); + HttpResponse response = handler.handle(request); + assertThat(response.getStatus(), is(OK)); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"test\":\"PUT received\"}")); + } + + @Test + public void invalidMethod() { + final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); + final HttpRequest request = HttpRequest.createTestRequest(URI, GET); + HttpResponse response = handler.handle(request); + assertThat(response.getStatus(), is(METHOD_NOT_ALLOWED)); + } + + + /** + * A handler that prepares a session given by an id in the request. + * + * @author musum + * @since 5.1.14 + */ + public static class SessionExampleHandler extends ThreadedHttpRequestHandler { + + public SessionExampleHandler(Executor executor) { + super(executor); + } + + @Override + public HttpResponse handle(HttpRequest request) { + final com.yahoo.jdisc.http.HttpRequest.Method method = request.getMethod(); + switch (method) { + case PUT: + return handlePUT(request); + case GET: + return new SessionExampleResponse(METHOD_NOT_ALLOWED, "Method '" + method + "' is not supported"); + default: + return new SessionExampleResponse(INTERNAL_SERVER_ERROR); + } + } + + @SuppressWarnings({"UnusedDeclaration"}) + HttpResponse handlePUT(HttpRequest request) { + return new SessionExampleResponse(OK, "PUT received"); + } + + private static class SessionExampleResponse extends HttpResponse { + private final Slime slime = new Slime(); + private final Cursor root = slime.setObject(); + private final String message; + + + private SessionExampleResponse(int status) { + this(status, ""); + headers().put("Cache-Control","max-age=120"); + } + + private SessionExampleResponse(int status, String message) { + super(status); + this.message = message; + } + + @Override + public void render(OutputStream outputStream) throws IOException { + root.setString("test", message); + new JsonFormat(true).encode(outputStream, slime); + } + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java new file mode 100644 index 00000000000..d38b7f9e586 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.transaction.Transaction; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.HostRegistry; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.session.*; + +import java.io.*; +import java.util.Optional; + +/** + * Base class for session handler tests + * + * @author musum + * @since 5.1.14 + */ +public class SessionHandlerTest { + + protected String pathPrefix = "/application/v2/session/"; + public static final String hostname = "foo"; + public static final int port = 1337; + + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath, InputStream data) { + return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method, data); + } + + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath) { + return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method); + } + + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id) { + return createTestRequest(path, method, cmd, id, ""); + } + + public static HttpRequest createTestRequest(String path) { + return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path, com.yahoo.jdisc.http.HttpRequest.Method.PUT); + } + + public static String getRenderedString(HttpResponse response) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + return baos.toString("UTF-8"); + } + + public static class MockSession extends LocalSession { + + public boolean doVerboseLogging = false; + public Session.Status status; + private final SessionPreparer preparer; + private final ApplicationPackage app; + private ConfigChangeActions actions = new ConfigChangeActions(); + private long createTime = System.currentTimeMillis() / 1000; + private ApplicationId applicationId; + + public MockSession(long id, ApplicationPackage app) { + super(TenantName.defaultName(), id, null, new SessionContext(null, new MockSessionZKClient(MockApplicationPackage.createEmpty()), null, null, new HostRegistry<>(), null)); + this.app = app; + this.preparer = new SessionTest.MockSessionPreparer(); + } + + public MockSession(long sessionId, ApplicationPackage applicationPackage, long createTime) { + this(sessionId, applicationPackage); + this.createTime = createTime; + } + + public MockSession(long sessionId, ApplicationPackage applicationPackage, ConfigChangeActions actions) { + this(sessionId, applicationPackage); + this.actions = actions; + } + + public MockSession(long sessionId, ApplicationPackage app, ApplicationId applicationId) { + this(sessionId, app); + this.applicationId = applicationId; + } + + @Override + public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath) { + status = Session.Status.PREPARE; + if (doVerboseLogging) { + logger.log(LogLevel.DEBUG, "debuglog"); + } + return actions; + } + + public void setStatus(Session.Status status) { + this.status = status; + } + + @Override + public Session.Status getStatus() { + return this.status; + } + + @Override + public Transaction createDeactivateTransaction() { + return new DummyTransaction().add((DummyTransaction.RunnableOperation) () -> { + status = Status.DEACTIVATE; + }); + } + + @Override + public Transaction createActivateTransaction() { + return new DummyTransaction().add((DummyTransaction.RunnableOperation) () -> { + status = Status.ACTIVATE; + }); + } + + @Override + public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { + if (mode == Mode.WRITE) { + status = Status.NEW; + } + if (preparer == null) { + return null; + } + ApplicationPackage pkg = app; + if (pkg == null) { + return null; + } + return pkg.getFile(relativePath); + } + + @Override + public ApplicationId getApplicationId() { + return applicationId; + } + + @Override + public long getCreateTime() { + return createTime; + } + + @Override + public void delete() { } + } + + public static enum Cmd { + PREPARED("prepared"), + ACTIVE("active"), + CONTENT("content"); + private final String name; + + private Cmd(String s) { + this.name = s; + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionPrepareHandlerTestBase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionPrepareHandlerTestBase.java new file mode 100644 index 00000000000..885d4164196 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionPrepareHandlerTestBase.java @@ -0,0 +1,185 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.session.*; + +import com.yahoo.vespa.curator.Curator; +import org.junit.Test; + +import java.io.IOException; + +import static com.yahoo.jdisc.http.HttpRequest.Method; +import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author musum + * @since 5.1.14 + */ +public abstract class SessionPrepareHandlerTestBase extends SessionHandlerTest { + + protected Curator curator; + private SessionZooKeeperClient zooKeeperClient; + protected LocalSessionRepo localRepo; + + protected String preparedMessage = " prepared.\"}"; + protected String tenantMessage = ""; + + + @Test + public void require_error_when_session_id_does_not_exist() throws Exception { + // No session with this id exists + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.PUT, Cmd.PREPARED, 9999L)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "Session 9999 was not found"); + } + + @Test + public void require_error_when_session_id_not_a_number() throws Exception { + final String session = "notanumber/prepared"; + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix + session)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, + HttpErrorResponse.errorCodes.BAD_REQUEST, + "Session id in request is not a number, request was 'http://" + hostname + ":" + port + pathPrefix + session + "'"); + } + + @Test + public void require_that_handler_gives_error_for_unsupported_methods() throws Exception { + testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, Method.POST, Cmd.PREPARED, 1L)); + testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, Method.DELETE, Cmd.PREPARED, 1L)); + } + + protected void testUnsupportedMethod(HttpRequest request) throws Exception { + HttpResponse response = createHandler().handle(request); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, METHOD_NOT_ALLOWED, + HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, + "Method '" + request.getMethod().name() + "' is not supported"); + } + + @Test + public void require_that_activate_url_is_returned_on_success() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.PUT, Cmd.PREPARED, 1L)); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + assertNotNull(response); + assertThat(response.getStatus(), is(OK)); + assertResponseContains(response, "\"activate\":\"http://foo:1337" + pathPrefix + "1/active\",\"message\":\"Session 1" + preparedMessage); + } + + @Test + public void require_debug() throws Exception { + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.PUT, Cmd.PREPARED, 9999L, "?debug=true")); + assertThat(response.getStatus(), is(NOT_FOUND)); + assertThat(SessionHandlerTest.getRenderedString(response), containsString("NotFoundException")); + } + + @Test + public void require_verbose() throws Exception { + MockSession session = new MockSession(1, null); + session.doVerboseLogging = true; + localRepo.addSession(session); + HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.PUT, Cmd.PREPARED, 1L, "?verbose=true")); + assertThat(response.getStatus(), is(OK)); + assertThat(SessionHandlerTest.getRenderedString(response), containsString("debuglog")); + } + + /** + * A mock remote session repo based on contents of local repo + */ + private RemoteSessionRepo fromLocalSessionRepo(LocalSessionRepo localRepo) { + RemoteSessionRepo remoteRepo = new RemoteSessionRepo(); + PathProvider pathProvider = new PathProvider(Path.createRoot()); + for (LocalSession ls : localRepo.listSessions()) { + + zooKeeperClient = new MockSessionZKClient(curator, pathProvider.getSessionDirs().append(String.valueOf(ls.getSessionId()))); + if (ls.getStatus()!=null) zooKeeperClient.writeStatus(ls.getStatus()); + RemoteSession remSess = new RemoteSession(TenantName.from("default"), ls.getSessionId(), + new TestComponentRegistry(), + zooKeeperClient); + remoteRepo.addSession(remSess); + } + return remoteRepo; + } + + @Test + public void require_get_response_activate_url_on_ok() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + SessionHandler sessHandler = createHandler(fromLocalSessionRepo(localRepo)); + sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.PUT, Cmd.PREPARED, 1L)); + session.setStatus(Session.Status.PREPARE); + zooKeeperClient.writeStatus(Session.Status.PREPARE); + HttpResponse getResponse = sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.GET, Cmd.PREPARED, 1L)); + assertResponseContains(getResponse, "\"activate\":\"http://foo:1337" + pathPrefix + "1/active\",\"message\":\"Session 1" + preparedMessage); + } + + @Test + public void require_get_response_error_on_not_prepared() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + SessionHandler sessHandler = createHandler(fromLocalSessionRepo(localRepo)); + session.setStatus(Session.Status.NEW); + zooKeeperClient.writeStatus(Session.Status.NEW); + HttpResponse getResponse = sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.GET, Cmd.PREPARED, 1L)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, + HttpErrorResponse.errorCodes.BAD_REQUEST, + "Session not prepared: 1"); + session.setStatus(Session.Status.ACTIVATE); + zooKeeperClient.writeStatus(Session.Status.ACTIVATE); + getResponse = sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.GET, Cmd.PREPARED, 1L)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, + HttpErrorResponse.errorCodes.BAD_REQUEST, + "Session is active: 1"); + } + + @Test + public void require_cannot_prepare_active_session() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + session.setStatus(Session.Status.ACTIVATE); + SessionHandler sessionHandler = createHandler(fromLocalSessionRepo(localRepo)); + HttpResponse putResponse = sessionHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.PUT, Cmd.PREPARED, 1L)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(putResponse, BAD_REQUEST, + HttpErrorResponse.errorCodes.BAD_REQUEST, + "Session is active: 1"); + } + + @Test + public void require_get_response_error_when_session_id_does_not_exist() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + SessionHandler sessHandler = createHandler(fromLocalSessionRepo(localRepo)); + HttpResponse getResponse = sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, Method.GET, Cmd.PREPARED, 9999L)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(getResponse, NOT_FOUND, + HttpErrorResponse.errorCodes.NOT_FOUND, + "Session 9999 was not found"); + } + + protected static void assertResponse(HttpResponse response, String activateString) throws IOException { + // TODO Test when more logging is added + //assertThat(baos.toString(), startsWith("{\"log\":[{\"time\":")); + assertThat(SessionHandlerTest.getRenderedString(response), endsWith(activateString)); + } + + protected static void assertResponseContains(HttpResponse response, String string) throws IOException { + assertThat(SessionHandlerTest.getRenderedString(response), containsString(string)); + } + + protected static void assertResponseNotContains(HttpResponse response, String string) throws IOException { + assertThat(SessionHandlerTest.getRenderedString(response), not(containsString(string))); + } + + public abstract SessionHandler createHandler() throws Exception; + + public abstract SessionHandler createHandler(RemoteSessionRepo remoteSessionRepo) throws Exception ; +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java new file mode 100644 index 00000000000..1e1bf1a644c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.Session; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class ApplicationContentHandlerTest extends ContentHandlerTestBase { + + private ApplicationHandler handler; + private TenantName tenant1 = TenantName.from("mofet"); + private TenantName tenant2 = TenantName.from("bla"); + private String baseServer = "http://foo:1337"; + + private ApplicationId idTenant1 = new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("foo").instanceName("quux").build(); + private ApplicationId idTenant2 = new ApplicationId.Builder() + .tenant(tenant2) + .applicationName("foo").instanceName("quux").build(); + private MockSession session2; + + @Before + public void setupHandler() throws Exception { + TestTenantBuilder testTenantBuilder = new TestTenantBuilder(); + testTenantBuilder.createTenant(tenant1); + testTenantBuilder.createTenant(tenant2); + session2 = new MockSession(2l, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); + testTenantBuilder.tenants().get(tenant1).getLocalSessionRepo().addSession(session2); + testTenantBuilder.tenants().get(tenant2).getLocalSessionRepo().addSession(new MockSession(3l, FilesApplicationPackage.fromFile(new File("src/test/apps/content2")))); + testTenantBuilder.tenants().get(tenant1).getApplicationRepo().createPutApplicationTransaction(idTenant1, 2l).commit(); + testTenantBuilder.tenants().get(tenant2).getApplicationRepo().createPutApplicationTransaction(idTenant2, 3l).commit(); + handler = new ApplicationHandler(command -> command.run(), AccessLog.voidAccessLog(), testTenantBuilder.createTenants(), HostProvisionerProvider.empty(), Zone.defaultZone(), null, null); + pathPrefix = createPath(idTenant1, Zone.defaultZone()); + baseUrl = baseServer + pathPrefix; + } + + private String createPath(ApplicationId applicationId, Zone zone) { + return "/application/v2/tenant/" + + applicationId.tenant().value() + + "/application/" + + applicationId.application().value() + + "/environment/" + + zone.environment().value() + + "/region/" + + zone.region().value() + + "/instance/" + + applicationId.instance().value() + + "/content/"; + } + + @Test + public void require_that_nonexistant_application_returns_not_found() throws IOException { + assertNotFound(HttpRequest.createTestRequest(baseServer + createPath(new ApplicationId.Builder() + .tenant("tenant") + .applicationName("notexist").instanceName("baz").build(), Zone.defaultZone()), + com.yahoo.jdisc.http.HttpRequest.Method.GET)); + assertNotFound(HttpRequest.createTestRequest(baseServer + createPath(new ApplicationId.Builder() + .tenant("unknown") + .applicationName("notexist").instanceName("baz").build(), Zone.defaultZone()), + com.yahoo.jdisc.http.HttpRequest.Method.GET)); + } + + @Test + public void require_that_multiple_tenants_are_handled() throws IOException { + assertContent("/test.txt", "foo\n"); + pathPrefix = createPath(idTenant2, Zone.defaultZone()); + baseUrl = baseServer + pathPrefix; + assertContent("/test.txt", "bar\n"); + } + + @Test + public void require_that_get_does_not_set_write_flag() throws IOException { + session2.status = Session.Status.PREPARE; + assertContent("/test.txt", "foo\n"); + assertThat(session2.status, is(Session.Status.PREPARE)); + } + + private void assertNotFound(HttpRequest request) { + HttpResponse response = handler.handle(request); + assertNotNull(response); + assertThat(response.getStatus(), is(Response.Status.NOT_FOUND)); + } + + @Override + protected HttpResponse doRequest(com.yahoo.jdisc.http.HttpRequest.Method method, String path) { + HttpRequest request = HttpRequest.createTestRequest(baseUrl + path, method); + return handler.handle(request); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java new file mode 100644 index 00000000000..1eb38902e3f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -0,0 +1,289 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.MockReloadHandler; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.Tenant; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; +import com.yahoo.vespa.config.server.http.HandlerTest; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.MockSessionZKClient; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.config.server.session.RemoteSessionRepo; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModelFactory; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import javax.ws.rs.client.Client; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author musum + * @since 5.4 + */ +public class ApplicationHandlerTest { + + private static File testApp = new File("src/test/apps/app"); + + private ApplicationHandler handler; + private ListApplicationsHandler listApplicationsHandler; + private final static TenantName mytenant = TenantName.from("mytenant"); + private final static TenantName foobar = TenantName.from("foobar"); + private Tenants tenants; + private SessionActiveHandlerTest.MockProvisioner provisioner; + private MockStateApiFactory stateApiFactory = new MockStateApiFactory(); + + @Before + public void setup() throws Exception { + TestTenantBuilder testBuilder = new TestTenantBuilder(); + testBuilder.createTenant(mytenant).withReloadHandler(new MockReloadHandler()); + testBuilder.createTenant(foobar).withReloadHandler(new MockReloadHandler()); + + tenants = testBuilder.createTenants(); + provisioner = new SessionActiveHandlerTest.MockProvisioner(); + handler = createApplicationHandler( + provisioner, new ApplicationConvergenceChecker(stateApiFactory), new LogServerLogGrabber()); + listApplicationsHandler = new ListApplicationsHandler( + Runnable::run, AccessLog.voidAccessLog(), tenants, Zone.defaultZone()); + } + + private ApplicationHandler createApplicationHandler( + Provisioner provisioner, + ApplicationConvergenceChecker convergeChecker, + LogServerLogGrabber logServerLogGrabber) { + return new ApplicationHandler( + Runnable::run, + AccessLog.voidAccessLog(), + tenants, + HostProvisionerProvider.withProvisioner(provisioner), + Zone.defaultZone(), + convergeChecker, + logServerLogGrabber); + } + + @Test + public void testDelete() throws Exception { + ApplicationId defaultId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenant).build(); + assertApplicationExists(mytenant, null, Zone.defaultZone()); + + long sessionId = 1; + addApplication(tenants.tenantsCopy().get(mytenant), defaultId, sessionId); + assertApplicationExists(mytenant, defaultId, Zone.defaultZone()); + assertFalse(provisioner.removed); + deleteAndAssertOKResponse(defaultId); + assertTrue(provisioner.removed); + assertThat(provisioner.lastApplicationId.tenant(), is(mytenant)); + assertThat(provisioner.lastApplicationId, is(defaultId)); + sessionId++; + addApplication(tenants.tenantsCopy().get(mytenant), defaultId, sessionId); + deleteAndAssertOKResponse(defaultId, true); + + ApplicationId fooId = new ApplicationId.Builder() + .tenant(mytenant) + .applicationName("foo").instanceName("quux").build(); + sessionId++; + addApplication(tenants.tenantsCopy().get(mytenant), fooId, sessionId); + addApplication(tenants.tenantsCopy().get(foobar), fooId, sessionId); + assertApplicationExists(mytenant, fooId, Zone.defaultZone()); + assertApplicationExists(foobar, fooId, Zone.defaultZone()); + deleteAndAssertOKResponse(fooId, true); + assertThat(provisioner.lastApplicationId.tenant(), is(mytenant)); + assertThat(provisioner.lastApplicationId, is(fooId)); + assertApplicationExists(mytenant, null, Zone.defaultZone()); + assertApplicationExists(foobar, fooId, Zone.defaultZone()); + + + sessionId++; + ApplicationId baliId = new ApplicationId.Builder() + .tenant(mytenant) + .applicationName("bali").instanceName("quux").build(); + addApplication(tenants.tenantsCopy().get(mytenant), baliId, sessionId); + deleteAndAssertOKResponse(baliId, true); + assertApplicationExists(mytenant, null, Zone.defaultZone()); + } + + @Test + public void testGet() throws Exception { + long sessionId = 1; + ApplicationId defaultId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenant).build(); + addApplication(tenants.tenantsCopy().get(mytenant), defaultId, sessionId); + assertApplicationGeneration(defaultId, Zone.defaultZone(), 1, true); + assertApplicationGeneration(defaultId, Zone.defaultZone(), 1, false); + } + + @Test + public void testRestart() throws Exception { + long sessionId = 1; + ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenant).build(); + addApplication(tenants.tenantsCopy().get(mytenant), application, sessionId); + assertFalse(provisioner.restarted); + restart(application, Zone.defaultZone()); + assertTrue(provisioner.restarted); + assertEquals(application, provisioner.lastApplicationId); + } + + @Test + public void testConverge() throws Exception { + long sessionId = 1; + ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenant).build(); + addApplication(tenants.tenantsCopy().get(mytenant), application, sessionId); + assertFalse(stateApiFactory.createdApi); + converge(application, Zone.defaultZone()); + assertTrue(stateApiFactory.createdApi); + } + + @Test + public void testPutIsIllegal() throws IOException { + assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.PUT); + } + + @Test + @Ignore + public void testFailingProvisioner() throws Exception { + provisioner = new SessionActiveHandlerTest.FailingMockProvisioner(); + handler = createApplicationHandler( + provisioner, new ApplicationConvergenceChecker(stateApiFactory), new LogServerLogGrabber()); + final ApplicationId applicationId = ApplicationId.defaultId(); + addApplication(tenants.tenantsCopy().get(mytenant), applicationId, 1); + assertApplicationExists(mytenant, applicationId, Zone.defaultZone()); + provisioner.activated = true; + + String url = "http://myhost:14000/application/v2/tenant/" + mytenant + "/application/" + applicationId.application(); + deleteAndAssertResponse(url, 500, null, "{\"message\":\"Cannot remove application\"}", com.yahoo.jdisc.http.HttpRequest.Method.DELETE); + assertApplicationExists(mytenant, applicationId, Zone.defaultZone()); + Assert.assertTrue(provisioner.activated); + } + + static void addApplication(Tenant tenant, ApplicationId applicationId, long sessionId) throws Exception { + tenant.getApplicationRepo().createPutApplicationTransaction(applicationId, sessionId).commit(); + ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); + addLocalSession(tenant, app, sessionId, applicationId); + addRemoteSession(tenant, app, sessionId); + } + + static void addLocalSession(Tenant tenant, ApplicationPackage app, long sessionId, ApplicationId applicationId) { + LocalSessionRepo localRepo = tenant.getLocalSessionRepo(); + localRepo.addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); + } + + static void addRemoteSession(Tenant tenant, ApplicationPackage app, long sessionId) { + RemoteSessionRepo remoteRepo = tenant.getRemoteSessionRepo(); + remoteRepo.addSession(new RemoteSession(tenant.getName(), sessionId, new TestComponentRegistry(new MockCurator(), new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))), new MockSessionZKClient(app))); + } + + private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { + String url = "http://myhost:14000/application/v2/tenant/" + mytenant + "/application/default"; + deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", + method); + } + + private void deleteAndAssertOKResponse(ApplicationId applicationId) throws IOException { + deleteAndAssertOKResponse(applicationId, true); + } + + private void deleteAndAssertOKResponse(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { + long sessionId = tenants.tenantsCopy().get(applicationId.tenant()).getApplicationRepo().getSessionIdForApplication(applicationId); + deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); + assertNull(tenants.tenantsCopy().get(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); + } + + private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException { + String expectedResponse = "{\"message\":\"Application '" + applicationId + "' deleted\"}"; + deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE); + } + + private void assertApplicationGeneration(ApplicationId applicationId, Zone zone, long expectedGeneration, boolean fullAppIdInUrl) throws IOException { + assertApplicationGeneration(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedGeneration); + } + + private String toUrlPath(ApplicationId application, Zone zone, boolean fullAppIdInUrl) { + String url = "http://myhost:14000/application/v2/tenant/" + application.tenant().value() + "/application/" + application.application().value(); + if (fullAppIdInUrl) + url = url + "/environment/" + zone.environment().value() + "/region/" + zone.region().value() + "/instance/" + application.instance().value(); + return url; + } + + private void assertApplicationGeneration(String url, long expectedGeneration) throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"generation\":" + expectedGeneration + "}"); + } + + private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method)); + if (expectedStatus == 200) { + HandlerTest.assertHttpStatusCodeAndMessage(response, 200, expectedResponse); + } else { + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, expectedStatus, errorCode, expectedResponse); + } + } + + private void assertApplicationExists(TenantName tenantName, ApplicationId applicationId, Zone zone) throws IOException { + String expected = applicationId == null ? "[]" : "[\"http://myhost:14000/application/v2/tenant/" + tenantName + "/application/" + applicationId.application().value() + + "/environment/" + zone.environment().value() + + "/region/" + zone.region().value() + + "/instance/" + applicationId.instance().value() + "\"]"; + ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/", + Response.Status.OK, + expected, + com.yahoo.jdisc.http.HttpRequest.Method.GET); + } + + private void restart(ApplicationId application, Zone zone) throws IOException { + String restartUrl = toUrlPath(application, zone, true) + "/restart"; + HttpResponse response = handler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.POST)); + HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); + } + + private void converge(ApplicationId application, Zone zone) throws IOException { + String restartUrl = toUrlPath(application, zone, true) + "/converge"; + HttpResponse response = handler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); + } + + private static class MockStateApiFactory implements ApplicationConvergenceChecker.StateApiFactory { + public boolean createdApi = false; + @Override + public ApplicationConvergenceChecker.StateApi createStateApi(Client client, URI serviceUri) { + createdApi = true; + return () -> { + try { + return new ObjectMapper().readTree("{\"config\":{\"generation\":1}}"); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java new file mode 100644 index 00000000000..50ef9176771 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.*; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.http.HandlerTest; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author musum + * @since 5.4 + */ +public class HostHandlerTest { + private static final String urlPrefix = "http://myhost:14000/application/v2/host/"; + + private HostHandler handler; + private final static TenantName mytenant = TenantName.from("mytenant"); + private final static String hostname = "testhost"; + private Tenants tenants; + private HostRegistries hostRegistries; + private HostHandler hostHandler; + + @Before + public void setup() throws Exception { + TestTenantBuilder testBuilder = new TestTenantBuilder(); + testBuilder.createTenant(mytenant).withReloadHandler(new MockReloadHandler()); + + tenants = testBuilder.createTenants(); + handler = createHostHandler(); + } + + private HostHandler createHostHandler() { + final HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + hostRegistry.update(mytenant, Collections.singletonList(hostname)); + TestComponentRegistry testComponentRegistry = new TestComponentRegistry(); + hostRegistries = testComponentRegistry.getHostRegistries(); + hostRegistries.createApplicationHostRegistry(mytenant).update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); + hostRegistries.getTenantHostRegistry().update(mytenant, Collections.singletonList(hostname)); + hostHandler = new HostHandler(command -> { + command.run(); + }, AccessLog.voidAccessLog(), testComponentRegistry); + return hostHandler; + } + + @Test + public void require_correct_tenant_and_application_for_hostname() throws Exception { + assertThat(hostRegistries, is(hostHandler.hostRegistries)); + long sessionId = 1; + ApplicationId id = ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()); + ApplicationHandlerTest.addApplication(tenants.tenantsCopy().get(mytenant), id, sessionId); + assertApplicationForHost(hostname, mytenant, id, Zone.defaultZone()); + } + + @Test + public void require_that_handler_gives_error_for_unknown_hostname() throws Exception { + long sessionId = 1; + ApplicationHandlerTest.addApplication(tenants.tenantsCopy().get(mytenant), ApplicationId.defaultId(), sessionId); + final String hostname = "unknown"; + assertErrorForHost(hostname, + Response.Status.NOT_FOUND, + HttpErrorResponse.errorCodes.NOT_FOUND, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not find any application using host '" + hostname + "'\"}"); + } + + @Test + public void require_that_only_get_method_is_allowed() throws IOException { + assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.PUT); + assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.POST); + assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.DELETE); + } + + private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { + String url = urlPrefix + hostname; + deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, + HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, + "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", + method); + } + + private void assertApplicationForHost(String hostname, TenantName expectedTenantName, ApplicationId expectedApplicationId, Zone zone) throws IOException { + String url = urlPrefix + hostname; + HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HandlerTest.assertHttpStatusCodeAndMessage(response, Response.Status.OK, + "{\"tenant\":\"" + expectedTenantName.value() + "\"," + + "\"application\":\"" + expectedApplicationId.application().value() + "\"," + + "\"environment\":\"" + zone.environment().value() + "\"," + + "\"region\":\"" + zone.region().value() + "\"," + + "\"instance\":\"" + expectedApplicationId.instance().value() + "\"}" + ); + } + + private void assertErrorForHost(String hostname, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse) throws IOException { + String url = urlPrefix + hostname; + HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, expectedStatus, errorCode, expectedResponse); + } + + private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, expectedStatus, errorCode, expectedResponse); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java new file mode 100644 index 00000000000..b1a330e5f99 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import java.io.IOException; +import java.io.StringReader; +import java.util.Collections; +import java.util.HashSet; +import java.util.concurrent.Executor; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import org.junit.Before; +import org.junit.Test; +import com.yahoo.config.SimpletypesConfig; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.protocol.SlimeConfigResponse; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.MockRequestHandler; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.http.HandlerTest; +import com.yahoo.vespa.config.server.http.HttpConfigRequest; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; + +public class HttpGetConfigHandlerTest { + + private static final TenantName tenant = TenantName.from("mytenant"); + private static final String EXPECTED_RENDERED_STRING = "{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"; + private static final String configUri = "http://yahoo.com:8080/config/v2/tenant/" + tenant.value() + "/application/myapplication/foo.bar/myid"; + private MockRequestHandler mockRequestHandler; + private HttpGetConfigHandler handler; + + @Before + public void setUp() throws Exception { + mockRequestHandler = new MockRequestHandler(); + mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>() {{ + add(new ConfigKey<>("bar", "myid", "foo")); + }} ); + TestTenantBuilder tb = new TestTenantBuilder(); + tb.createTenant(tenant).withRequestHandler(mockRequestHandler).build(); + Tenants tenants = tb.createTenants(); + handler = new HttpGetConfigHandler(command -> { + command.run(); + }, AccessLog.voidAccessLog(), tenants); + } + + @Test + public void require_that_handler_can_be_created() throws IOException { + // Define config response for mock handler + final long generation = 1L; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + InnerCNode targetDef = getInnerCNode(); + mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), + SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); + assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); + } + + @Test + public void require_that_handler_can_handle_long_appid_request_with_configid() throws IOException { + String uriLongAppId = "http://yahoo.com:8080/config/v2/tenant/" + tenant.value() + + "/application/myapplication/environment/staging/region/myregion/instance/myinstance/foo.bar/myid"; + final long generation = 1L; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + InnerCNode targetDef = getInnerCNode(); + mockRequestHandler.responses.put(new ApplicationId.Builder() + .tenant(tenant) + .applicationName("myapplication").instanceName("myinstance").build(), + SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET)); + assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); + } + + @Test + public void require_that_request_gets_correct_fields_with_full_appid() { + String uriLongAppId = "http://yahoo.com:8080/config/v2/tenant/bill/application/sookie/environment/dev/region/bellefleur/instance/sam/foo.bar/myid"; + HttpRequest r = HttpRequest.createTestRequest(uriLongAppId, GET); + HttpConfigRequest req = HttpConfigRequest.createFromRequestV2(r); + assertThat(req.getApplicationId().tenant().value(), is("bill")); + assertThat(req.getApplicationId().application().value(), is("sookie")); + assertThat(req.getApplicationId().instance().value(), is("sam")); + } + + @Test + public void require_that_request_gets_correct_fields_with_short_appid() { + String uriShortAppId = "http://yahoo.com:8080/config/v2/tenant/jason/application/alcide/foo.bar/myid"; + HttpRequest r = HttpRequest.createTestRequest(uriShortAppId, GET); + HttpConfigRequest req = HttpConfigRequest.createFromRequestV2(r); + assertThat(req.getApplicationId().tenant().value(), is("jason")); + assertThat(req.getApplicationId().application().value(), is("alcide")); + assertThat(req.getApplicationId().instance().value(), is("default")); + } + + @Test + public void require_correct_error_response() throws IOException { + final String nonExistingConfigNameUri = "http://yahoo.com:8080/config/v2/tenant/mytenant/application/myapplication/nonexisting.config/myid"; + final String nonExistingConfigUri = "http://yahoo.com:8080/config/v2/tenant/mytenant/application/myapplication//foo.bar/myid/nonexisting/id"; + final String illegalConfigNameUri = "http://yahoo.com:8080/config/v2/tenant/mytenant/application/myapplication//foobar/myid"; + + HttpResponse response = handler.handle(HttpRequest.createTestRequest(nonExistingConfigNameUri, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config: nonexisting.config"); + assertTrue(SessionHandlerTest.getRenderedString(response).contains("No such config:")); + response = handler.handle(HttpRequest.createTestRequest(nonExistingConfigUri, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config id: myid/nonexisting/id"); + assertEquals(response.getContentType(), "application/json"); + assertTrue(SessionHandlerTest.getRenderedString(response).contains("No such config id:")); + response = handler.handle(HttpRequest.createTestRequest(illegalConfigNameUri, GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Illegal config, must be of form namespace.name."); + } + + @Test + public void require_that_nocache_property_works() throws IOException { + long generation = 1L; + ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); + InnerCNode targetDef = getInnerCNode(); + mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), + SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, "mymd5")); + final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); + HttpResponse response = handler.handle(request); + assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); + } + + private InnerCNode getInnerCNode() { + // TODO: Hope to be able to remove this mess soon. + DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); + return dParser.getTree(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java new file mode 100644 index 00000000000..cad7cbd583c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -0,0 +1,145 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.MockRequestHandler; +import com.yahoo.vespa.config.server.Tenants; +import com.yahoo.vespa.config.server.http.HandlerTest; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.http.v2.HttpListConfigsHandler.ListConfigsResponse; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; + +/** + * @author lulf + * @since 5.1 + */ +public class HttpListConfigsHandlerTest { + + private MockRequestHandler mockRequestHandler; + private HttpListConfigsHandler handler; + private HttpListNamedConfigsHandler namedHandler; + + @Before + public void setUp() throws Exception { + mockRequestHandler = new MockRequestHandler(); + mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>() {{ + add(new ConfigKey<>("bar", "conf/id", "foo")); + }} ); + TestTenantBuilder tb = new TestTenantBuilder(); + tb.createTenant(TenantName.from("mytenant")).withRequestHandler(mockRequestHandler).build(); + Tenants tenants = tb.createTenants(); + handler = new HttpListConfigsHandler(command -> { + command.run(); + }, AccessLog.voidAccessLog(), tenants, Zone.defaultZone()); + namedHandler = new HttpListNamedConfigsHandler(command -> { + command.run(); + }, AccessLog.voidAccessLog(), tenants, Zone.defaultZone()); + } + + @Test + public void require_that_handler_can_be_created() throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest("http://yahoo.com:8080/config/v2/tenant/mytenant/application/myapplication/", GET)); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"children\":[],\"configs\":[]}")); + } + + @Test + public void require_that_request_can_be_created_from_full_appid() throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest( + "http://yahoo.com:8080/config/v2/tenant/mytenant/application/myapplication/environment/test/region/myregion/instance/myinstance/", GET)); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"children\":[],\"configs\":[]}")); + } + + @Test + public void require_that_named_handler_can_be_created() throws IOException { + HttpRequest req = HttpRequest.createTestRequest("http://foo.com:8080/config/v2/tenant/mytenant/application/myapplication/foo.bar/conf/id/", GET); + req.getJDiscRequest().parameters().put("http.path", Arrays.asList("foo.bar")); + HttpResponse response = namedHandler.handle(req); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"children\":[],\"configs\":[]}")); + } + + @Test + public void require_that_named_handler_can_be_created_from_full_appid() throws IOException { + HttpRequest req = HttpRequest.createTestRequest("http://foo.com:8080/config/v2/tenant/mytenant/application/myapplication/environment/prod/region/myregion/instance/myinstance/foo.bar/conf/id/", GET); + req.getJDiscRequest().parameters().put("http.path", Arrays.asList("foo.bar")); + HttpResponse response = namedHandler.handle(req); + assertThat(SessionHandlerTest.getRenderedString(response), is("{\"children\":[],\"configs\":[]}")); + } + + @Test + public void require_child_listings_correct() { + Set<ConfigKey<?>> keys = new LinkedHashSet<ConfigKey<?>>() {{ + add(new ConfigKey<>("name1", "id/1", "ns1")); + add(new ConfigKey<>("name1", "id/1", "ns1")); + add(new ConfigKey<>("name1", "id/2", "ns1")); + add(new ConfigKey<>("name1", "", "ns1")); + add(new ConfigKey<>("name1", "id/1/1", "ns1")); + add(new ConfigKey<>("name1", "id2", "ns1")); + add(new ConfigKey<>("name1", "id/2/1", "ns1")); + add(new ConfigKey<>("name1", "id/2/1/5/6", "ns1")); + }}; + Set<ConfigKey<?>> keysThatHaveChild = ListConfigsResponse.keysThatHaveAChildWithSameName(keys, keys); + assertEquals(keysThatHaveChild.size(), 3); + } + + @Test + public void require_url_building_and_mimetype_correct() { + ListConfigsResponse resp = new ListConfigsResponse(new HashSet<ConfigKey<?>>(), null, "http://foo.com/config/v2/tenant/mytenant/application/mya/", true); + assertEquals(resp.toUrl(new ConfigKey<>("myconfig", "my/id", "mynamespace"), true), "http://foo.com/config/v2/tenant/mytenant/application/mya/mynamespace.myconfig/my/id"); + assertEquals(resp.toUrl(new ConfigKey<>("myconfig", "my/id", "mynamespace"), false), "http://foo.com/config/v2/tenant/mytenant/application/mya/mynamespace.myconfig/my/id/"); + assertEquals(resp.toUrl(new ConfigKey<>("myconfig", "", "mynamespace"), false), "http://foo.com/config/v2/tenant/mytenant/application/mya/mynamespace.myconfig"); + assertEquals(resp.toUrl(new ConfigKey<>("myconfig", "", "mynamespace"), true), "http://foo.com/config/v2/tenant/mytenant/application/mya/mynamespace.myconfig"); + assertEquals(resp.getContentType(), "application/json"); + + } + + @Test + public void require_error_on_bad_request() throws IOException { + HttpRequest req = HttpRequest.createTestRequest("http://foo.com:8080/config/v2/tenant/mytenant/application/myapplication/foobar/conf/id/", GET); + HttpResponse resp = namedHandler.handle(req); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(resp, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Illegal config, must be of form namespace.name."); + req = HttpRequest.createTestRequest("http://foo.com:8080/config/v2/tenant/mytenant/application/myapplication/foo.barNOPE/conf/id/", GET); + resp = namedHandler.handle(req); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(resp, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config: foo.barNOPE"); + req = HttpRequest.createTestRequest("http://foo.com:8080/config/v2/tenant/mytenant/application/myapplication/foo.bar/conf/id/NOPE/", GET); + resp = namedHandler.handle(req); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(resp, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "No such config id: conf/id/NOPE"); + } + + @Test + public void require_correct_error_response_on_no_model() throws IOException { + mockRequestHandler.setAllConfigs(new HashSet<ConfigKey<?>>()); + HttpResponse response = namedHandler.handle(HttpRequest.createTestRequest("http://yahoo.com:8080/config/v2/tenant/mytenant/application/myapplication/foo.bar/myid/", GET)); + HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, + HttpErrorResponse.errorCodes.NOT_FOUND, + "Config not available, verify that an application package has been deployed and activated."); + } + + @Test + public void require_correct_configid_parent() { + assertEquals(ListConfigsResponse.parentConfigId(null), null); + assertEquals(ListConfigsResponse.parentConfigId("foo"), ""); + assertEquals(ListConfigsResponse.parentConfigId(""), ""); + assertEquals(ListConfigsResponse.parentConfigId("/"), ""); + assertEquals(ListConfigsResponse.parentConfigId("foo/bar"), "foo"); + assertEquals(ListConfigsResponse.parentConfigId("foo/bar/baz"), "foo/bar"); + assertEquals(ListConfigsResponse.parentConfigId("foo/bar/"), "foo/bar"); + + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java new file mode 100644 index 00000000000..2e75d1b02e6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.*; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import org.junit.Test; +import org.junit.Before; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import static com.yahoo.jdisc.http.HttpRequest.Method.*; + +/** + * @author lulf + * @since 5.1 + */ +public class ListApplicationsHandlerTest { + private ApplicationRepo applicationRepo, applicationRepo2; + private ListApplicationsHandler handler; + + @Before + public void setup() throws Exception { + TestTenantBuilder testBuilder = new TestTenantBuilder(); + TenantName mytenant = TenantName.from("mytenant"); + TenantName foobar = TenantName.from("foobar"); + testBuilder.createTenant(mytenant); + testBuilder.createTenant(foobar); + applicationRepo = testBuilder.tenants().get(mytenant).getApplicationRepo(); + applicationRepo2 = testBuilder.tenants().get(foobar).getApplicationRepo(); + Tenants tenants = testBuilder.createTenants(); + handler = new ListApplicationsHandler(command -> { + command.run(); + }, AccessLog.voidAccessLog(), tenants, new Zone(Environment.dev, RegionName.from("us-east"))); + } + + @Test + public void require_that_applications_are_listed() throws Exception { + final String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; + assertResponse(url, Response.Status.OK, + "[]"); + applicationRepo.createPutApplicationTransaction( + new ApplicationId.Builder().tenant("tenant").applicationName("foo").instanceName("quux").build(), + 1).commit(); + assertResponse(url, Response.Status.OK, + "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); + applicationRepo.createPutApplicationTransaction( + new ApplicationId.Builder().tenant("tenant").applicationName("bali").instanceName("quux").build(), + 1).commit(); + assertResponse(url, Response.Status.OK, + "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"," + + "\"" + url + "bali/environment/dev/region/us-east/instance/quux\"]" + ); + } + + @Test + public void require_that_get_is_required() throws IOException { + final String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; + assertResponse(url, Response.Status.METHOD_NOT_ALLOWED, + createMethodNotAllowedMessage(DELETE), DELETE); + assertResponse(url, Response.Status.METHOD_NOT_ALLOWED, + createMethodNotAllowedMessage(PUT), PUT); + assertResponse(url, Response.Status.METHOD_NOT_ALLOWED, + createMethodNotAllowedMessage(POST), POST); + } + + private static String createMethodNotAllowedMessage(Method method) { + return "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method.name() + "' is not supported\"}"; + } + + @Test + public void require_that_listing_works_with_multiple_tenants() throws Exception { + applicationRepo.createPutApplicationTransaction(new ApplicationId.Builder() + .tenant("tenant") + .applicationName("foo").instanceName("quux").build(), 1).commit(); + applicationRepo2.createPutApplicationTransaction(new ApplicationId.Builder() + .tenant("tenant") + .applicationName("quux").instanceName("foo").build(), 1).commit(); + String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; + assertResponse(url, Response.Status.OK, + "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); + url = "http://myhost:14000/application/v2/tenant/foobar/application/"; + assertResponse(url, Response.Status.OK, + "[\"" + url + "quux/environment/dev/region/us-east/instance/foo\"]"); + } + + void assertResponse(String url, int expectedStatus, String expectedResponse) throws IOException { + assertResponse(url, expectedStatus, expectedResponse, GET); + } + + private void assertResponse(String url, int expectedStatus, String expectedResponse, Method method) throws IOException { + assertResponse(handler, url, expectedStatus, expectedResponse, method); + } + + static void assertResponse(ListApplicationsHandler handler, String url, int expectedStatus, String expectedResponse, Method method) throws IOException { + HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method)); + assertNotNull(response); + assertThat(response.getStatus(), is(expectedStatus)); + assertThat(SessionHandlerTest.getRenderedString(response), is(expectedResponse)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java new file mode 100644 index 00000000000..9bcc462035a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponseTest.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.TenantName; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +public class ListTenantsResponseTest extends TenantTest { + + private final TenantName a = TenantName.from("a"); + private final TenantName b = TenantName.from("b"); + private final TenantName c = TenantName.from("c"); + + @Test + public void testJsonSerialization() throws Exception { + final Collection<TenantName> tenantNames = Arrays.asList(a, b, c); + final ListTenantsResponse response = new ListTenantsResponse(tenantNames); + assertResponseEquals(response, "{\"tenants\":[\"a\",\"b\",\"c\"]}"); + } + + @Test + public void testJsonSerializationNoTenants() throws Exception { + final Collection<TenantName> tenantNames = Collections.emptyList(); + final ListTenantsResponse response = new ListTenantsResponse(tenantNames); + assertResponseEquals(response, "{\"tenants\":[]}"); + } +}
\ No newline at end of file diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java new file mode 100644 index 00000000000..e4f985b7d9f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListTenantsTest.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.TestWithTenant; +import org.junit.Test; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.http.HttpRequest.Method; + +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.assertTrue; + +public class ListTenantsTest extends TenantTest { + + private final TenantName a = TenantName.from("a"); + private final TenantName b = TenantName.from("b"); + private final TenantName c = TenantName.from("c"); + + @Test + public void testListTenants() throws Exception { + tenants.createTenant(a); + tenants.createTenant(b); + tenants.createTenant(c); + + ListTenantsHandler listTenantsHandler = new ListTenantsHandler(testExecutor(), null, tenants); + + ListTenantsResponse response = (ListTenantsResponse) listTenantsHandler.handleGET(HttpRequest.createTestRequest("/blabla", Method.GET)); + final Collection<TenantName> responseTenantNames = response.getTenantNames(); + assertTrue(responseTenantNames.containsAll(Arrays.asList(a, b, c))); + assertContainsSystemTenants(responseTenantNames); + } + + private static void assertContainsSystemTenants(final Collection<TenantName> tenantNames) { + assertTrue(tenantNames.contains(TenantName.defaultName())); + assertTrue(tenantNames.contains(ApplicationId.HOSTED_VESPA_TENANT)); + } + + @Test + public void testEmptyTenants() throws Exception { + ListTenantsHandler listTenantsHandler = new ListTenantsHandler(testExecutor(), null, tenants); + + ListTenantsResponse response = (ListTenantsResponse) listTenantsHandler.handleGET(HttpRequest.createTestRequest("/blabla", Method.GET)); + final Collection<TenantName> responseTenantNames = response.getTenantNames(); + assertContainsSystemTenants(responseTenantNames); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java new file mode 100644 index 00000000000..659a435ca6d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -0,0 +1,217 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executor; + +import com.yahoo.config.provision.*; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.slime.JsonFormat; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.config.server.session.RemoteSessionRepo; +import com.yahoo.vespa.config.server.session.Session; +import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; + +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.Before; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; + +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.http.SessionActiveHandlerTestBase; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.SessionCreateHandlerTestBase.MockSessionFactory; + +public class SessionActiveHandlerTest extends SessionActiveHandlerTestBase { + + private MockProvisioner hostProvisioner; + + @Before + public void setup() throws Exception { + tenant = TenantName.from("activatetest"); + remoteSessionRepo = new RemoteSessionRepo(); + applicationRepo = new MemoryApplicationRepo(); + curator = new MockCurator(); + configCurator = ConfigCurator.create(curator); + localRepo = new LocalSessionRepo(applicationRepo); + pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; + tenantMessage = ",\"tenant\":\"" + tenant + "\""; + pathProvider = new PathProvider(Path.createRoot()); + activatedMessage = " for tenant '" + tenant + "' activated."; + hostProvisioner = new MockProvisioner(); + } + + @Test + public void require_correct_response_on_success() throws Exception { + activateAndAssertOK(1, 0); + } + + @Test + public void testActivationWithActivationInBetween() throws Exception { + activateAndAssertOK(90l, 0l); + activateAndAssertError(92l, 89l, + HttpErrorResponse.errorCodes.BAD_REQUEST, + "tenant:"+tenant+" app:default:default Cannot activate session 92 because the currently active session (90) has changed since session 92 was created (was 89 at creation time)"); + } + + + @Test + public void testActivationOfUnpreparedSession() throws Exception { + // Needed so we can test that previous active session is still active after a failed activation + RemoteSession firstSession = activateAndAssertOK(90l, 0l); + long sessionId = 91l; + ActivateRequest activateRequest = new ActivateRequest(sessionId, 0l, Session.Status.NEW, "").invoke(); + HttpResponse actResponse = activateRequest.getActResponse(); + RemoteSession session = activateRequest.getSession(); + assertThat(actResponse.getStatus(), is(Response.Status.BAD_REQUEST)); + assertThat(getRenderedString(actResponse), is("{\"error-code\":\"BAD_REQUEST\",\"message\":\"tenant:"+tenant+" app:default:default Session " + sessionId + " is not prepared\"}")); + assertThat(session.getStatus(), is(not(Session.Status.ACTIVATE))); + assertThat(firstSession.getStatus(), is(Session.Status.ACTIVATE)); + } + + @Test + public void require_that_handler_gives_error_for_unsupported_methods() throws Exception { + testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.POST, Cmd.PREPARED, 1L)); + testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.DELETE, Cmd.PREPARED, 1L)); + testUnsupportedMethod(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L)); + } + + @Test + @Ignore + public void require_that_handler_gives_error_when_provisioner_activated_fails() throws Exception { + hostProvisioner = new FailingMockProvisioner(); + hostProvisioner.activated = false; + activateAndAssertError(1, 0, HttpErrorResponse.errorCodes.BAD_REQUEST, "Cannot activate application"); + assertFalse(hostProvisioner.activated); + } + + @Override + protected RemoteSession activateAndAssertOK(long sessionId, long previousSessionId) throws Exception { + ActivateRequest activateRequest = activateAndAssertOKPut(sessionId, previousSessionId, ""); + return activateRequest.getSession(); + } + + @Override + protected Session activateAndAssertOK(long sessionId, long previousSessionId, String subPath) throws Exception { + ActivateRequest activateRequest = activateAndAssertOKPut(sessionId, previousSessionId, subPath); + return activateRequest.getSession(); + } + + @Override + protected void assertActivationMessageOK(ActivateRequest activateRequest, String message) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + new JsonFormat(true).encode(byteArrayOutputStream, activateRequest.getMetaData().getSlime()); + assertThat(message, containsString("\"tenant\":\"" + tenant + "\",\"message\":\"Session " + activateRequest.getSessionId() + activatedMessage)); + assertThat(message, containsString("/application/v2/tenant/" + tenant + + "/application/" + appName + + "/environment/" + "prod" + + "/region/" + "default" + + "/instance/" + "default")); + assertTrue(hostProvisioner.activated); + assertThat(hostProvisioner.lastHosts.size(), is(1)); + } + + @Override + protected void activateAndAssertError(long sessionId, long previousSessionId, HttpErrorResponse.errorCodes errorCode, String expectedError) throws Exception { + hostProvisioner.activated = false; + activateAndAssertErrorPut(sessionId, previousSessionId, errorCode, expectedError); + assertFalse(hostProvisioner.activated); + } + + @Override + protected void writeApplicationId(SessionZooKeeperClient zkc, String applicationName) { + ApplicationId id = ApplicationId.from(tenant, + ApplicationName.from(applicationName), InstanceName.defaultName()); + zkc.writeApplicationId(id); + } + + @Override + protected String getActivateLogPre() { + return "tenant:testtenant, app:default:default "; + } + + @Override + protected SessionHandler createHandler() throws Exception { + final MockSessionFactory sessionFactory = new MockSessionFactory(); + TestTenantBuilder testTenantBuilder = new TestTenantBuilder(); + testTenantBuilder.createTenant(tenant) + .withSessionFactory(sessionFactory) + .withLocalSessionRepo(localRepo) + .withRemoteSessionRepo(remoteSessionRepo) + .withApplicationRepo(applicationRepo) + .build(); + return new SessionActiveHandler(new Executor() { + @SuppressWarnings("NullableProblems") + @Override + public void execute(Runnable command) { + command.run(); + } + }, AccessLog.voidAccessLog(), testTenantBuilder.createTenants(), HostProvisionerProvider.withProvisioner(hostProvisioner), Zone.defaultZone()); + } + + public static class MockProvisioner implements Provisioner { + + boolean activated = false; + boolean removed = false; + boolean restarted = false; + ApplicationId lastApplicationId; + Collection<HostSpec> lastHosts; + + @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + throw new UnsupportedOperationException(); + } + + @Override + public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { + transaction.commit(); + activated = true; + lastApplicationId = application; + lastHosts = hosts; + } + + @Override + public void removed(ApplicationId application) { + removed = true; + lastApplicationId = application; + } + + @Override + public void restart(ApplicationId application, HostFilter filter) { + restarted = true; + lastApplicationId = application; + } + + } + + public static class FailingMockProvisioner extends MockProvisioner { + + @Override + public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { + throw new IllegalArgumentException("Cannot activate application"); + } + + @Override + public void removed(ApplicationId application) { + throw new IllegalArgumentException("Cannot remove application"); + } + + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java new file mode 100644 index 00000000000..3ca2f2304cc --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.vespa.config.server.http.SessionContentHandlerTestBase; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import org.junit.Before; + +import java.io.InputStream; +import java.util.concurrent.Executor; + +/** + * @author lulf + * @since 5.1 + */ +public class SessionContentHandlerTest extends SessionContentHandlerTestBase { + private static final TenantName tenant = TenantName.from("contenttest"); + private SessionContentHandler handler = null; + + @Before + public void setupHandler() throws Exception { + handler = createHandler(); + pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; + baseUrl = "http://foo:1337/application/v2/tenant/" + tenant + "/session/1/content/"; + } + + protected HttpResponse doRequest(HttpRequest.Method method, String path) { + return doRequest(method, path, 1l); + } + + protected HttpResponse doRequest(HttpRequest.Method method, String path, long sessionId) { + return handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, method, Cmd.CONTENT, sessionId, path)); + } + + protected HttpResponse doRequest(HttpRequest.Method method, String path, InputStream data) { + return doRequest(method, path, 1l, data); + } + + protected HttpResponse doRequest(HttpRequest.Method method, String path, long sessionId, InputStream data) { + return handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, method, Cmd.CONTENT, sessionId, path, data)); + } + + private SessionContentHandler createHandler() throws Exception { + TestTenantBuilder testTenantBuilder = new TestTenantBuilder(); + testTenantBuilder.createTenant(tenant).getLocalSessionRepo().addSession(new MockSession(1l, FilesApplicationPackage.fromFile(createTestApp()))); + return new SessionContentHandler(new Executor() { + @SuppressWarnings("NullableProblems") + @Override + public void execute(Runnable command) { + command.run(); + } + }, AccessLog.voidAccessLog(), testTenantBuilder.createTenants()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java new file mode 100644 index 00000000000..d31cdc1d1e1 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.config.server.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.http.SessionCreateHandlerTestBase; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.session.*; +import org.junit.Before; +import org.junit.Test; + +import java.io.*; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executor; + +import static org.junit.Assert.*; + +import static com.yahoo.jdisc.http.HttpRequest.Method.*; + +/** + * @author musum + * @since 5.1 + */ +public class SessionCreateHandlerTest extends SessionCreateHandlerTestBase { + + private static final TenantName tenant = TenantName.from("test"); + + @Before + public void setupRepo() throws Exception { + applicationRepo = new MemoryApplicationRepo(); + localSessionRepo = new LocalSessionRepo(applicationRepo); + pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; + createdMessage = " for tenant '" + tenant + "' created.\""; + tenantMessage = ",\"tenant\":\"test\""; + } + + @Test + public void require_that_application_urls_can_be_given_as_from_parameter() throws Exception { + localSessionRepo.addSession(new SessionHandlerTest.MockSession(2l, FilesApplicationPackage.fromFile(testApp))); + ApplicationId fooId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("foo") + .instanceName("quux") + .build(); + applicationRepo.createPutApplicationTransaction(fooId, 2).commit(); + assertFromParameter("3", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux"); + localSessionRepo.addSession(new SessionHandlerTest.MockSession(5l, FilesApplicationPackage.fromFile(testApp))); + ApplicationId bioId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("foobio") + .instanceName("quux") + .build(); + applicationRepo.createPutApplicationTransaction(bioId, 5).commit(); + assertFromParameter("6", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foobio/environment/staging/region/baz/instance/quux"); + } + + @Test + public void require_that_from_parameter_must_be_valid() throws IOException { + assertIllegalFromParameter("active"); + assertIllegalFromParameter(""); + assertIllegalFromParameter("http://host:4013/application/v2/tenant/" + tenant + "/application/lol"); + assertIllegalFromParameter("http://host:4013/application/v2/tenant/" + tenant + "/application/foo/environment/prod"); + assertIllegalFromParameter("http://host:4013/application/v2/tenant/" + tenant + "/application/foo/environment/prod/region/baz"); + assertIllegalFromParameter("http://host:4013/application/v2/tenant/" + tenant + "/application/foo/environment/prod/region/baz/instance"); + } + + @Override + public SessionCreateHandler createHandler() { + try { + return createHandler(new MockSessionFactory()); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + return null; + } + + @Override + public SessionCreateHandler createHandler(SessionFactory sessionFactory) { + try { + TestTenantBuilder testBuilder = new TestTenantBuilder(); + testBuilder.createTenant(tenant).withSessionFactory(sessionFactory) + .withLocalSessionRepo(localSessionRepo) + .withApplicationRepo(applicationRepo); + return createHandler(testBuilder.createTenants()); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + SessionCreateHandler createHandler(Tenants tenants) { + return new SessionCreateHandler(new Executor() { + @SuppressWarnings("NullableProblems") + @Override + public void execute(Runnable command) { + command.run(); + } + }, AccessLog.voidAccessLog(), tenants, new ConfigserverConfig(new ConfigserverConfig.Builder())); + } + + @Override + public HttpRequest post() throws FileNotFoundException { + return post(null, postHeaders, new HashMap<String, String>()); + } + + @Override + public HttpRequest post(File file) throws FileNotFoundException { + return post(file, postHeaders, new HashMap<String, String>()); + } + + @Override + public HttpRequest post(File file, Map<String, String> headers, Map<String, String> parameters) throws FileNotFoundException { + HttpRequest request = HttpRequest.createTestRequest("http://" + hostname + ":" + port + "/application/v2/tenant/" + tenant + "/session", + POST, + file == null ? null : new FileInputStream(file), + parameters); + for (Map.Entry<String, String> entry : headers.entrySet()) { + request.getJDiscRequest().headers().put(entry.getKey(), entry.getValue()); + } + return request; + } + + @Override + public HttpRequest post(Map<String, String> parameters) throws FileNotFoundException { + return post(null, new HashMap<String, String>(), parameters); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java new file mode 100644 index 00000000000..0a5d4a1843c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -0,0 +1,246 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.path.Path; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.HostRegistry; +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.configchange.MockRefeedAction; +import com.yahoo.vespa.config.server.configchange.MockRestartAction; +import com.yahoo.vespa.config.server.http.*; +import com.yahoo.vespa.config.server.session.*; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executor; + +import static com.yahoo.jdisc.Response.Status.OK; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author musum + * + * @since 5.1.14 + */ +public class SessionPrepareHandlerTest extends SessionPrepareHandlerTestBase { + private static final TenantName tenant = TenantName.from("test"); + private TestTenantBuilder builder; + + @Before + public void setupRepo() throws Exception { + ApplicationRepo applicationRepo = new MemoryApplicationRepo(); + curator = new MockCurator(); + localRepo = new LocalSessionRepo(applicationRepo); + pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; + preparedMessage = " for tenant '" + tenant + "' prepared.\""; + tenantMessage = ",\"tenant\":\"" + tenant + "\""; + builder = new TestTenantBuilder(); + } + + @Test + public void require_that_tenant_is_in_response() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + HttpResponse response = createHandler(addTestTenant()).handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + assertNotNull(response); + assertThat(response.getStatus(), is(OK)); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + assertResponseContains(response, tenantMessage); + } + + @Test + public void require_that_preparing_with_multiple_tenants_work() throws Exception { + ApplicationRepo applicationRepoDefault = new MemoryApplicationRepo(); + LocalSessionRepo localRepoDefault = new LocalSessionRepo(applicationRepoDefault); + final TenantName tenantName = TenantName.defaultName(); + addTenant(tenantName, localRepoDefault, new RemoteSessionRepo(), new SessionCreateHandlerTestBase.MockSessionFactory()); + addTestTenant(); + final SessionHandler handler = createHandler(builder); + + long sessionId = 1; + // Deploy with default tenant + MockSession session = new MockSession(sessionId, null); + localRepoDefault.addSession(session); + pathPrefix = "/application/v2/tenant/default/session/"; + + HttpResponse response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); + assertNotNull(response); + assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + + // Same session id, as this is for another tenant + session = new MockSession(sessionId, null); + localRepo.addSession(session); + String applicationName = "myapp"; + pathPrefix = "/application/v2/tenant/" + tenant + "/session/" + sessionId + "/prepared?applicationName=" + applicationName; + response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix)); + assertNotNull(response); + assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + + sessionId++; + session = new MockSession(sessionId, null); + localRepo.addSession(session); + pathPrefix = "/application/v2/tenant/" + tenant + "/session/" + sessionId + "/prepared?applicationName=" + applicationName + "&instance=quux"; + response = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix)); + assertNotNull(response); + assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + } + + @Test + public void require_that_config_change_actions_are_in_response() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + HttpResponse response = createHandler(addTestTenant()).handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + assertResponseContains(response, "\"configChangeActions\":{\"restart\":[],\"refeed\":[]}"); + } + + @Test + public void require_that_config_change_actions_are_logged_if_existing() throws Exception { + List<ServiceInfo> services = Collections.singletonList(new ServiceInfo("serviceName", "serviceType", null, + ImmutableMap.of("clustername", "foo", "clustertype", "bar"), "configId", "hostName")); + ConfigChangeActions actions = new ConfigChangeActions(Arrays.asList( + new MockRestartAction("change", services), + new MockRefeedAction("change-id", false, "other change", services, "test"))); + MockSession session = new MockSession(1, null, actions); + localRepo.addSession(session); + HttpResponse response = createHandler(addTestTenant()).handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + assertResponseContains(response, "Change(s) between active and new application that require restart:\\nIn cluster 'foo' of type 'bar"); + assertResponseContains(response, "Change(s) between active and new application that may require re-feed:\\nchange-id: Consider removing data and re-feed document type 'test'"); + } + + @Test + public void require_that_config_change_actions_are_not_logged_if_not_existing() throws Exception { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + HttpResponse response = createHandler(addTestTenant()).handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + assertResponseNotContains(response, "Change(s) between active and new application that require restart"); + assertResponseNotContains(response, "Change(s) between active and new application that require re-feed"); + } + + @Test + public void test_out_of_capacity_response() throws InterruptedException, IOException { + String message = "No nodes available"; + SessionThrowingException session = new SessionThrowingException(new OutOfCapacityException(message)); + localRepo.addSession(session); + HttpResponse response = createHandler(addTestTenant()).handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + assertEquals(400, response.getStatus()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + Slime data = new Slime(); + new JsonDecoder().decode(data, baos.toByteArray()); + assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.OUT_OF_CAPACITY.name())); + assertThat(data.get().field("message").asString(), is(message)); + } + + @Override + public SessionHandler createHandler() throws Exception { + return createHandler(addTestTenant()); + } + + @Override + public SessionHandler createHandler(RemoteSessionRepo remoteSessionRepo) throws Exception { + return createHandler(addTenant(tenant, localRepo, remoteSessionRepo, + new SessionCreateHandlerTestBase.MockSessionFactory())); + } + + private TestTenantBuilder addTestTenant() { + return addTenant(tenant, localRepo, new RemoteSessionRepo(), + new SessionCreateHandlerTestBase.MockSessionFactory()); + } + + static SessionHandler createHandler(TestTenantBuilder builder) { + return new SessionPrepareHandler(new Executor() { + @SuppressWarnings("NullableProblems") + @Override + public void execute(Runnable command) { + command.run(); + }}, AccessLog.voidAccessLog(), builder.createTenants(), new ConfigserverConfig(new ConfigserverConfig.Builder())); + } + + private TestTenantBuilder addTenant(TenantName tenantName, + LocalSessionRepo localSessionRepo, + RemoteSessionRepo remoteSessionRepo, + SessionFactory sessionFactory) { + builder.createTenant(tenantName).withSessionFactory(sessionFactory) + .withLocalSessionRepo(localSessionRepo) + .withRemoteSessionRepo(remoteSessionRepo) + .withApplicationRepo(new MemoryApplicationRepo()); + return builder; + } + + public static class SessionThrowingException extends LocalSession { + private final RuntimeException exception; + + public SessionThrowingException(RuntimeException exception) { + super(TenantName.defaultName(), 1, null, new SessionContext(null, new MockSessionZKClient(MockApplicationPackage.createEmpty()), null, null, new HostRegistry<>(), null)); + this.exception = exception; + } + + @Override + public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath) { + throw exception; + } + + @Override + public Session.Status getStatus() { + return null; + } + + @Override + public Transaction createDeactivateTransaction() { + return null; + } + + @Override + public Transaction createActivateTransaction() { + return null; + } + + @Override + public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { + return null; + } + + @Override + public ApplicationId getApplicationId() { + return null; + } + + @Override + public long getCreateTime() { + return 0; + } + + @Override + public void delete() { } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java new file mode 100644 index 00000000000..106b675c2c7 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +import java.io.IOException; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.config.server.*; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.NotFoundException; + +public class TenantHandlerTest extends TenantTest { + + private TenantHandler handler; + private final TenantName a = TenantName.from("a"); + + @Before + public void setup() throws Exception { + handler = new TenantHandler(testExecutor(), null, tenants); + } + + @Test + public void testTenantCreate() throws Exception { + assertFalse(tenants.tenantsCopy().containsKey(a)); + TenantCreateResponse response = (TenantCreateResponse) putSync(a, + HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.PUT)); + assertResponseEquals(response, "{\"message\":\"Tenant a created.\"}"); + } + + @Test + public void testTenantCreateWithAllPossibleCharactersInName() throws Exception { + TenantName tenantName = TenantName.from("aB-9999_foo"); + assertFalse(tenants.tenantsCopy().containsKey(tenantName)); + TenantCreateResponse response = (TenantCreateResponse) putSync(a, + HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/" + tenantName, Method.PUT)); + assertResponseEquals(response, "{\"message\":\"Tenant " + tenantName + " created.\"}"); + } + + private HttpResponse putSync(TenantName name, HttpRequest testRequest) throws InterruptedException { + HttpResponse response = handler.handlePUT(testRequest); + return response; + } + + @Test(expected=NotFoundException.class) + public void testGetNonExisting() throws Exception { + handler.handleGET(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/x", Method.GET)); + } + + @Test + public void testGetExisting() throws Exception { + tenants.createTenant(a); + TenantGetResponse response = (TenantGetResponse) handler.handleGET(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.GET)); + assertResponseEquals(response, "{\"message\":\"Tenant 'a' exists.\"}"); + } + + @Test(expected=BadRequestException.class) + public void testCreateExisting() throws Exception { + assertFalse(tenants.tenantsCopy().containsKey(a)); + TenantCreateResponse response = (TenantCreateResponse) putSync(a, HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.PUT)); + assertResponseEquals(response, "{\"message\":\"Tenant a created.\"}"); + Tenant ta = tenants.tenantsCopy().get(a); + assertEquals(ta.getName(), a); + handler.handlePUT(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.PUT)); + } + + @Test + public void testDelete() throws IOException, InterruptedException { + putSync(a, HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.PUT)); + assertEquals(tenants.tenantsCopy().get(a).getName(), a); + TenantDeleteResponse delResp = (TenantDeleteResponse) handler.handleDELETE(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.DELETE)); + assertResponseEquals(delResp, "{\"message\":\"Tenant a deleted.\"}"); + assertFalse(tenants.tenantsCopy().containsKey(a)); + } + + @Test + public void testDeleteTenantWithActiveApplications() throws Exception { + putSync(a, HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/" + a, Method.PUT)); + assertEquals(tenants.tenantsCopy().get(a).getName(), a); + + final Tenant tenant = tenants.tenantsCopy().get(a); + final int sessionId = 1; + ApplicationId app = ApplicationId.from(a, + ApplicationName.from("foo"), InstanceName.defaultName()); + ApplicationHandlerTest.addApplication(tenant, app, sessionId); + + try { + handler.handleDELETE(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/" + a, Method.DELETE)); + fail(); + } catch (BadRequestException e) { + assertThat(e.getMessage(), is("Cannot delete tenant 'a', as it has active applications: [tenant 'a', application 'foo', instance 'default']")); + } + } + + @Test(expected=NotFoundException.class) + public void testDeleteNonExisting() { + handler.handleDELETE(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/x", Method.DELETE)); + } + + @Test(expected=BadRequestException.class) + public void testIllegalNameSlashes() throws InterruptedException { + putSync(a, HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a/b", Method.PUT)); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java new file mode 100644 index 00000000000..930787361af --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.Executor; + +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import org.junit.After; +import org.junit.Before; + +/** + * Supertype for tests in the multi tenant application API + * + * @author vegardh + * + */ +public class TenantTest extends TestWithCurator { + + protected Tenants tenants; + + @Before + public void setupTenants() throws Exception { + tenants = createTenants(); + } + + @After + public void closeTenants() throws IOException { + tenants.close(); + } + + protected Tenants createTenants() throws Exception { + return new Tenants(new TestComponentRegistry(curator), Metrics.createTestMetrics()); + } + + protected Executor testExecutor() { + return new Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }; + } + + protected void assertResponseEquals(SessionResponse response, String payload) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + assertEquals(baos.toString("UTF-8"), payload); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java new file mode 100644 index 00000000000..a128fa6c891 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TestTenantBuilder.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v2; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.http.SessionCreateHandlerTestBase; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.RemoteSessionRepo; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.util.*; + +/** + * Test utility for creating tenants used for testing and setup wiring of tenant stuff. + * + * @author lulf + * @since 5.1 + */ +public class TestTenantBuilder { + + private GlobalComponentRegistry componentRegistry; + private Map<TenantName, TenantBuilder> tenantMap = new HashMap<>(); + + public TestTenantBuilder() throws Exception { + componentRegistry = new TestComponentRegistry(new MockCurator()); + } + + public TenantBuilder createTenant(TenantName tenantName) { + MemoryApplicationRepo applicationRepo = new MemoryApplicationRepo(); + TenantBuilder builder = TenantBuilder.create(componentRegistry, tenantName, Path.createRoot().append(tenantName.value())) + .withSessionFactory(new SessionCreateHandlerTestBase.MockSessionFactory()) + .withLocalSessionRepo(new LocalSessionRepo(applicationRepo)) + .withRemoteSessionRepo(new RemoteSessionRepo()) + .withApplicationRepo(applicationRepo); + tenantMap.put(tenantName, builder); + return builder; + } + + public Map<TenantName, TenantBuilder> tenants() { + return Collections.unmodifiableMap(tenantMap); + } + + public Tenants createTenants() { + Collection<Tenant> tenantList = Collections2.transform(tenantMap.values(), new Function<TenantBuilder, Tenant>() { + @Override + public Tenant apply(TenantBuilder builder) { + try { + return builder.build(); + } catch (Exception e) { + throw new IllegalArgumentException("Unable to build tenant", e); + } + } + }); + return new Tenants(componentRegistry, Metrics.createTestMetrics(), tenantList); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java new file mode 100644 index 00000000000..d35f7abac5f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -0,0 +1,206 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.26 + */ +public class LbServicesProducerTest { + private static final String rotation1 = "rotation-1"; + private static final String rotation2 = "rotation-2"; + private static final String rotationString = rotation1 + "," + rotation2; + private static final Set<Rotation> rotations = Collections.singleton(new Rotation(rotationString)); + + @Test + public void testDeterministicGetConfig() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); + LbServicesConfig last = null; + for (int i = 0; i < 100; i++) { + testModel = randomizeTenant(testModel, i); + LbServicesConfig config = getLbServicesConfig(Zone.defaultZone(), testModel); + if (last != null) { + assertConfig(last, config); + } + last = config; + } + } + + @Test + public void testConfigAliases() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder()); + LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel); + final LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services("qrserver"); + assertThat(services.servicealiases().size(), is(1)); + assertThat(services.endpointaliases().size(), is(2)); + + assertThat(services.servicealiases(0), is("service1")); + assertThat(services.endpointaliases(0), is("foo1.bar1.com")); + assertThat(services.endpointaliases(1), is("foo2.bar2.com")); + } + + @Test + public void testConfigActiveRotation() throws IOException, SAXException { + { + RegionName regionName = RegionName.from("us-east-1"); + LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); + assertTrue(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").activeRotation()); + } + + { + RegionName regionName = RegionName.from("us-east-2"); + LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); + assertFalse(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").activeRotation()); + } + } + + private LbServicesConfig createModelAndGetLbServicesConfig(RegionName regionName) throws IOException, SAXException { + final Zone zone = new Zone(Environment.prod, regionName); + Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder(). + properties(new DeployProperties.Builder().zone(zone).build())); + return getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); + } + + private LbServicesConfig getLbServicesConfig(Zone zone, Map<TenantName, Map<ApplicationId, Application>> testModel) { + LbServicesProducer producer = new LbServicesProducer(testModel, zone); + LbServicesConfig.Builder builder = new LbServicesConfig.Builder(); + producer.getConfig(builder); + return new LbServicesConfig(builder); + } + + @Test + public void testConfigAliasesWithRotations() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); + RegionName regionName = RegionName.from("us-east-1"); + LbServicesConfig conf = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); + final LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").hosts("foo.foo.yahoo.com").services("qrserver"); + assertThat(services.servicealiases().size(), is(1)); + assertThat(services.endpointaliases().size(), is(4)); + + assertThat(services.servicealiases(0), is("service1")); + assertThat(services.endpointaliases(0), is("foo1.bar1.com")); + assertThat(services.endpointaliases(1), is("foo2.bar2.com")); + assertThat(services.endpointaliases(2), is(rotation1)); + assertThat(services.endpointaliases(3), is(rotation2)); + } + + private Map<TenantName, Map<ApplicationId, Application>> randomizeTenant(Map<TenantName, Map<ApplicationId, Application>> testModel, int seed) { + Map<TenantName, Map<ApplicationId, Application>> randomizedTenants = new LinkedHashMap<>(); + List<TenantName> keys = new ArrayList<>(testModel.keySet()); + Collections.shuffle(keys, new Random(seed)); + for (TenantName key : keys) { + randomizedTenants.put(key, randomizeApplications(testModel.get(key), randomizedTenants.size())); + } + return randomizedTenants; + } + + private Map<ApplicationId, Application> randomizeApplications(Map<ApplicationId, Application> applicationIdApplicationMap, int seed) { + Map<ApplicationId, Application> randomizedApplications = new LinkedHashMap<>(); + List<ApplicationId> keys = new ArrayList<>(applicationIdApplicationMap.keySet()); + Collections.shuffle(keys, new Random(seed)); + for (ApplicationId key : keys) { + randomizedApplications.put(key, applicationIdApplicationMap.get(key)); + } + return randomizedApplications; + } + + private Map<TenantName, Map<ApplicationId, Application>> createTestModel(DeployState.Builder deployStateBuilder) throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> tMap = new LinkedHashMap<>(); + TenantName foo = TenantName.from("foo"); + TenantName bar = TenantName.from("bar"); + TenantName baz = TenantName.from("baz"); + tMap.put(foo, createTestApplications(foo, deployStateBuilder)); + tMap.put(bar, createTestApplications(bar, deployStateBuilder)); + tMap.put(baz, createTestApplications(baz, deployStateBuilder)); + return tMap; + } + + private Map<ApplicationId, Application> createTestApplications(TenantName tenant, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + Map<ApplicationId, Application> aMap = new LinkedHashMap<>(); + ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(); + ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build(); + ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").build(); + aMap.put(fooApp, createApplication(fooApp, deploystateBuilder)); + aMap.put(barApp, createApplication(barApp, deploystateBuilder)); + aMap.put(bazApp, createApplication(bazApp, deploystateBuilder)); + return aMap; + } + + private Application createApplication(ApplicationId appId, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + return new Application(createVespaModel(createApplicationPackage( + appId.tenant() + "." + appId.application() + ".yahoo.com", appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"), + deploystateBuilder), + new ServerCache(), + 3l, + Version.fromIntValues(1, 2, 3), + MetricUpdater.createTestUpdater(), + appId); + } + + private ApplicationPackage createApplicationPackage(String host1, String host2) { + String hosts = "<hosts><host name='" + host1 + "'><alias>node1</alias></host><host name='" + host2 + "'><alias>node2</alias></host></hosts>"; + String services = "<services><admin version='2.0'><adminserver hostalias='node1' /><logserver hostalias='node1' /><slobroks><slobrok hostalias='node1' /><slobrok hostalias='node2' /></slobroks></admin>" + + "<jdisc id='mydisc' version='1.0'>" + + " <aliases>" + + " <endpoint-alias>foo2.bar2.com</endpoint-alias>" + + " <service-alias>service1</service-alias>" + + " <endpoint-alias>foo1.bar1.com</endpoint-alias>" + + " </aliases>" + + " <nodes>" + + " <node hostalias='node1' />" + + " </nodes>" + + " <search/>" + + "</jdisc>" + + "</services>"; + String deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod global-service-id='mydisc'>" + + " <region active='true'>us-east-1</region>" + + " <region active='false'>us-east-2</region>" + + " </prod>" + + "</deployment>"; + + return new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withDeploymentInfo(deploymentInfo).build(); + } + + private Model createVespaModel(ApplicationPackage applicationPackage, DeployState.Builder deployStateBuilder) throws IOException, SAXException { + return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build()); + } + + private void assertConfig(LbServicesConfig expected, LbServicesConfig actual) { + assertFalse(expected.toString().isEmpty()); + assertFalse(actual.toString().isEmpty()); + assertThat(expected.toString(), is(actual.toString())); + assertThat(ConfigPayload.fromInstance(expected).toString(true), is(ConfigPayload.fromInstance(actual).toString(true))); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java new file mode 100755 index 00000000000..a8263cd361a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; + +import com.yahoo.cloud.config.RoutingConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author can + */ +public class RoutingProducerTest { + @Test + public void testNodesFromRoutingAppOnly() throws Exception { + Map<TenantName, Map<ApplicationId, Application>> testModel = createTestModel(new DeployState.Builder()); + RoutingProducer producer = new RoutingProducer(testModel); + RoutingConfig.Builder builder = new RoutingConfig.Builder(); + producer.getConfig(builder); + RoutingConfig config = new RoutingConfig(builder); + assertThat(config.hosts().size(), is(2)); + assertThat(config.hosts(0), is("hosted-vespa.routing.yahoo.com")); + assertThat(config.hosts(1), is("hosted-vespa.routing2.yahoo.com")); + } + + private Map<TenantName, Map<ApplicationId, Application>> createTestModel(DeployState.Builder deployStateBuilder) throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> tMap = new LinkedHashMap<>(); + TenantName foo = TenantName.from("foo"); + TenantName bar = TenantName.from("bar"); + TenantName routing = TenantName.from(ApplicationId.HOSTED_VESPA_TENANT.value()); + tMap.put(foo, createTestApplications(foo, deployStateBuilder)); + tMap.put(bar, createTestApplications(bar, deployStateBuilder)); + tMap.put(routing, createTestApplications(routing, deployStateBuilder)); + return tMap; + } + + private Map<ApplicationId, Application> createTestApplications(TenantName tenant, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + Map<ApplicationId, Application> aMap = new LinkedHashMap<>(); + ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(); + ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build(); + ApplicationId routingApp = new ApplicationId.Builder().tenant(tenant).applicationName(ApplicationId.ROUTING_APPLICATION.value()).build(); + aMap.put(fooApp, createApplication(fooApp, deploystateBuilder)); + aMap.put(barApp, createApplication(barApp, deploystateBuilder)); + aMap.put(routingApp, createApplication(routingApp, deploystateBuilder)); + return aMap; + } + + private Application createApplication(ApplicationId appId, DeployState.Builder deploystateBuilder) throws IOException, SAXException { + return new Application(createVespaModel(createApplicationPackage( + appId.tenant() + "." + appId.application() + ".yahoo.com", appId.tenant().value() + "." + appId.application().value() + "2.yahoo.com"), + deploystateBuilder), + new ServerCache(), + 3l, + Version.fromIntValues(1, 2, 3), + MetricUpdater.createTestUpdater(), + appId); + } + + private ApplicationPackage createApplicationPackage(String host1, String host2) { + String hosts = "<hosts><host name='" + host1 + "'><alias>node1</alias></host><host name='" + host2 + "'><alias>node2</alias></host></hosts>"; + String services = "<services><admin version='2.0'><adminserver hostalias='node1' /><logserver hostalias='node1' /><slobroks><slobrok hostalias='node1' /><slobrok hostalias='node2' /></slobroks></admin>" + + "<jdisc id='mydisc' version='1.0'>" + + " <aliases>" + + " <endpoint-alias>foo2.bar2.com</endpoint-alias>" + + " <service-alias>service1</service-alias>" + + " <endpoint-alias>foo1.bar1.com</endpoint-alias>" + + " </aliases>" + + " <nodes>" + + " <node hostalias='node1' />" + + " </nodes>" + + " <search/>" + + "</jdisc>" + + "</services>"; + String deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod global-service-id='mydisc'>" + + " <region active='true'>us-east</region>" + + " </prod>" + + "</deployment>"; + + return new MockApplicationPackage.Builder() + .withHosts(hosts) + .withServices(services) + .withDeploymentInfo(deploymentInfo) + .build(); + } + + private Model createVespaModel(ApplicationPackage applicationPackage, DeployState.Builder deployStateBuilder) throws IOException, SAXException { + return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java new file mode 100644 index 00000000000..32d1b610194 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.model; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.model.VespaModelFactory; + +/** + * @author lulf + */ +public class TestModelFactory extends VespaModelFactory { + private final Version vespaVersion; + private ModelContext modelContext; + + public TestModelFactory(Version vespaVersion) { + super(new NullConfigModelRegistry()); + this.vespaVersion = vespaVersion; + } + + // Needed for testing (to get hold of ModelContext) + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + this.modelContext = modelContext; + return super.createAndValidateModel(modelContext, ignoreValidationErrors); + } + + @Override + public Version getVersion() { + return vespaVersion; + } + + public ModelContext getModelContext() { + return modelContext; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg new file mode 100644 index 00000000000..d3970ee48eb --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg @@ -0,0 +1,44 @@ +include: search/cluster.music +include: search/cluster.music +c 2 +storage[2] +storage[0].feeder[1] +storage[0].feeder[0] "test" +storage[1].feeder[2] +storage[1].feeder[0] "me" +storage[1].feeder[1] now +storage[1].id :parent: +storage[1].id2 pjatt +testref :parent: +testref2 some/babbel +config[1] +config[0].role "rtx" +#config[0].usewrapper false +config[0].id search/cluster.music/rtx/0 +f[1] +f[0].a "A" +f[0].b "B" +f[0].c "C" +f[0].h "H" +f[0].f "F" +f[0].notindef "notindef" +routingtable[1] +routingtable[0].hop[3] +routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" +routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" +routingtable[0].hop[1].name "search/cluster.music" +routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" +routingtable[0].hop[1].recipient[1] +routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" +routingtable[0].hop[2].selector "[DocumentRouteSelector]" +routingtable[0].hop[2].name "indexing" +routingtable[0].hop[2].notindef "not in def" +routingtable[0].hop[2].recipient[1] +routingtable[0].hop[2].recipient[0] "search/cluster.music" +notindef "dfsd" +nopenotindef[0] "boo" +nadaindef[0].naw 98 +mode NOTINDEF +rangecheck1 100 +rangecheck2 10000 +rangecheck3 20 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg new file mode 100644 index 00000000000..727a5052ed6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg @@ -0,0 +1,2 @@ +include: search/cluster.sports +c 67 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg new file mode 100644 index 00000000000..f4996027f60 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg @@ -0,0 +1 @@ +model vespa diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg new file mode 100644 index 00000000000..d75d76810f9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg @@ -0,0 +1,2 @@ +foo "bar" +gaz -78 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg new file mode 100644 index 00000000000..7ccdb73eb9a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg @@ -0,0 +1,2 @@ +include: search/cluster.logical/* +include: search/cluster.video/* diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg new file mode 100644 index 00000000000..5b07d3a2890 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg @@ -0,0 +1 @@ +include: search/cluster.music diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg new file mode 100644 index 00000000000..038d655e83c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg @@ -0,0 +1,13 @@ +directory /home/vespa/var/spool/vespa/ +keepsuccess false +parsers[4] +parsers[0].classname com.yahoo.vespaspooler.XMLFileParser +parsers[0].parameters[0] +parsers[1].classname com.yahoo.mail.vespa.spooler.MailFileParser +parsers[1].parameters[0] +parsers[2].classname com.yahoo.mail.vespa.spooler.UserDeleteParser +parsers[2].parameters[0] +parsers[3].classname com.yahoo.mail.vespa.spooler.VespaGrimParser +parsers[3].parameters[1] +parsers[3].parameters[0].key chunksize +parsers[3].parameters[0].value 5 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java new file mode 100644 index 00000000000..3831f94a77d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.provision; + +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * @author lulf + */ +public class StaticProvisionerTest { + @Test + public void sameHostsAreProvisioned() throws IOException, SAXException { + ApplicationPackage app = FilesApplicationPackage.fromFile(new File("src/test/apps/hosted")); + InMemoryProvisioner inMemoryHostProvisioner = new InMemoryProvisioner(false, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com", "host4.yahoo.com"); + VespaModel firstModel = createModel(app, inMemoryHostProvisioner); + + StaticProvisioner staticProvisioner = new StaticProvisioner(firstModel.getProvisionInfo().get()); + VespaModel secondModel = createModel(app, staticProvisioner); + + assertModelConfig(firstModel, secondModel); + } + + private void assertModelConfig(VespaModel firstModel, VespaModel secondModel) { + String firstConfig = getModelConfig(firstModel); + String secondConfig = getModelConfig(secondModel); + assertEquals(firstConfig, secondConfig); + } + + private String getModelConfig(VespaModel model) { + return ConfigPayload.fromInstance(model.getConfig(ModelConfig.class, "")).toString(); + } + + private VespaModel createModel(ApplicationPackage app, HostProvisioner provisioner) throws IOException, SAXException { + DeployState deployState = new DeployState.Builder() + .applicationPackage(app) + .modelHostProvisioner(provisioner) + .properties(new DeployProperties.Builder() + .multitenant(true) + .hostedVespa(true) + .build()) + .build(); + return new VespaModel(new NullConfigModelRegistry(), deployState); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/restapi/impl/StatusResourceTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/restapi/impl/StatusResourceTest.java new file mode 100644 index 00000000000..333cac4fd48 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/restapi/impl/StatusResourceTest.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.restapi.impl; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.restapi.resources.StatusInformation; +import com.yahoo.vespa.defaults.Defaults; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class StatusResourceTest { + @Test + public void require_that_status_handler_responds_to_ping() throws IOException { + StatusResource handler = new StatusResource(null, null, null, null, null, null, null, new TestComponentRegistry()); + assertNotNull(handler.getStatus().configserverConfig); + } + + @Test + public void require_that_generated_config_is_converted() { + ConfigserverConfig orig = new ConfigserverConfig(new ConfigserverConfig.Builder()); + StatusInformation.ConfigserverConfig conv = new StatusInformation.ConfigserverConfig(orig); + assertThat(conv.applicationDirectory, is(Defaults.getDefaults().underVespaHome(orig.applicationDirectory()))); + assertThat(conv.configModelPluginDir.size(), is(orig.configModelPluginDir().size())); + assertThat(conv.zookeeeperserver.size(), is(orig.zookeeperserver().size())); + assertThat(conv.zookeeperBarrierTimeout, is(orig.zookeeper().barrierTimeout())); + assertThat(conv.configServerDBDir, is(Defaults.getDefaults().underVespaHome(orig.configServerDBDir()))); + assertThat(conv.masterGeneration, is(orig.masterGeneration())); + assertThat(conv.maxgetconfigclients, is(orig.maxgetconfigclients())); + assertThat(conv.multitenant, is(orig.multitenant())); + assertThat(conv.numDelayedResponseThreads, is(orig.numDelayedResponseThreads())); + assertThat(conv.numthreads, is(orig.numthreads())); + assertThat(conv.payloadCompressionType, is(orig.payloadCompressionType())); + assertThat(conv.rpcport, is(orig.rpcport())); + assertThat(conv.sessionLifetime, is(orig.sessionLifetime())); + assertThat(conv.zookeepercfg, is(Defaults.getDefaults().underVespaHome(orig.zookeepercfg()))); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg new file mode 100644 index 00000000000..f3acd4cf8b9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg @@ -0,0 +1,5 @@ +asyncfetchocc 9 +d 3 +kanon -78.56 + +partialsd "sd" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg new file mode 100644 index 00000000000..5d8a01a18ea --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg @@ -0,0 +1,2 @@ +d 89 +search[1].feeder[1] "sportsfeeder1" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg new file mode 100644 index 00000000000..f6c35df398d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg @@ -0,0 +1 @@ +gaff -89 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg new file mode 100644 index 00000000000..c3d9b1e45a1 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg @@ -0,0 +1,8 @@ +classes[1] +classes[logical].id 1906788747 +classes[logical].name logical +classes[logical].fields[2] +classes[logical].fields[0].name sddocnameNAM +classes[logical].fields[0].type longstring +classes[logical].fields[1].name title +classes[logical].fields[1].type longstringSTRIN diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg new file mode 100644 index 00000000000..12a21671b4a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg @@ -0,0 +1,8 @@ +classes[1] +classes[music].id 1906788746 +classes[music].name music +classes[music].fields[2] +classes[music].fields[0].name sddocnameNAME +classes[music].fields[0].type longstring +classes[music].fields[1].name title +classes[music].fields[1].type longstringSTRING diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg new file mode 100644 index 00000000000..4001c59adbc --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg @@ -0,0 +1,14 @@ +classes[0] +classes[smallsum614540714].id 614540714 +classes[smallsum614540714].name smallsum +classes[smallsum614540714].fields[5] +classes[smallsum614540714].fields[0].name s_13 +classes[smallsum614540714].fields[0].type longstring +classes[smallsum614540714].fields[1].name ranklog +classes[smallsum614540714].fields[1].type longstring +classes[smallsum614540714].fields[2].name rankfeatures +classes[smallsum614540714].fields[2].type longstring +classes[smallsum614540714].fields[3].name summaryfeatures +classes[smallsum614540714].fields[3].type longstring +classes[smallsum614540714].fields[4].name sddocname +classes[smallsum614540714].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg new file mode 100644 index 00000000000..33d07b99ab6 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg @@ -0,0 +1,14 @@ +classes[0] +classes[smallsum507688128].id 507688128 +classes[smallsum507688128].name smallsum +classes[smallsum507688128].fields[5] +classes[smallsum507688128].fields[0].name title +classes[smallsum507688128].fields[0].type longstring +classes[smallsum507688128].fields[1].name ranklog +classes[smallsum507688128].fields[1].type longstring +classes[smallsum507688128].fields[2].name rankfeatures +classes[smallsum507688128].fields[2].type longstring +classes[smallsum507688128].fields[3].name summaryfeatures +classes[smallsum507688128].fields[3].type longstring +classes[smallsum507688128].fields[4].name sddocname +classes[smallsum507688128].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg new file mode 100644 index 00000000000..de9fbdd39f4 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg @@ -0,0 +1,7 @@ +rec 56 +national 77 +ilscript[1] +ilscript[music].name music +ilscript[music].doctype music +ilscript[music].content[1] +ilscript[music].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_4; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg new file mode 100644 index 00000000000..e95f976a43a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg @@ -0,0 +1,7 @@ +ursive -50 +teatern 78 +ilscript[1] +ilscript[father].name father +ilscript[father].doctype father +ilscript[father].content[6] +ilscript[father].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_5; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg new file mode 100644 index 00000000000..cea943d5bc9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg @@ -0,0 +1,2 @@ +include: search/cluster.music/conf1.sd.derived +include: search/cluster.music/conf2.sd.derived diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/DummyTransaction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/DummyTransaction.java new file mode 100644 index 00000000000..ea4455ab99f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/DummyTransaction.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.transaction.Transaction; + +import java.util.ArrayList; +import java.util.List; + +/** + * Dummy transaction implementation that only does stuff in memory and does not adhere to contract. + * @author lulf + */ +public class DummyTransaction implements Transaction { + + private final List<Operation> operations = new ArrayList<>(); + + public interface RunnableOperation extends Operation, Runnable { + } + + public DummyTransaction() { } + + @Override + public Transaction add(Operation operation) { + this.operations.add(operation); + return this; + } + + @Override + public Transaction add(List<Operation> operations) { + this.operations.addAll(operations); + return this; + } + + @Override + public List<Operation> operations() { return new ArrayList<>(operations); } + + @Override + public void prepare() { } + + @Override + public void commit() { + for (Operation op : operations) { + ((RunnableOperation)op).run(); + } + } + + @Override + public void rollbackOrLog() { + throw new IllegalStateException("Unexpected rollback"); + } + + @Override + public void close() { } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java new file mode 100644 index 00000000000..84fce1c09fe --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.path.Path; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.config.server.*; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; + +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.time.Duration; +import java.time.Instant; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * @author lulf + * @since 5.1 + */ +public class LocalSessionRepoTest extends TestWithCurator { + + private File testApp = new File("src/test/apps/app"); + private LocalSessionRepo repo; + private ManualClock clock; + private static final TenantName tenantName = TenantName.defaultName(); + + @Before + public void setupSessions() throws Exception { + setupSessions(tenantName, true); + } + + private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { + GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry(curator); + TenantFileSystemDirs tenantFileSystemDirs = TenantFileSystemDirs.createTestDirs(tenantName); + if (createInitialSessions) { + IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.path(), "1")); + IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.path(), "2")); + IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.path(), "3")); + } + clock = new ManualClock(Instant.ofEpochSecond(1)); + LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, + new SessionCounter(globalComponentRegistry.getCurator(), + Path.fromString("counter"), + Path.fromString("sessions")), + Path.createRoot(), + new MemoryApplicationRepo(), + tenantFileSystemDirs, new HostRegistry<>(), + tenantName); + repo = new LocalSessionRepo(tenantFileSystemDirs, loader, new MemoryApplicationRepo(), clock, 5); + } + + @Test + public void require_that_sessions_can_be_loaded_from_disk() { + assertNotNull(repo.getSession(1l)); + assertNotNull(repo.getSession(2l)); + assertNotNull(repo.getSession(3l)); + assertNull(repo.getSession(4l)); + } + + @Test + public void require_that_old_sessions_are_purged() { + clock.advance(Duration.ofSeconds(1)); + assertNotNull(repo.getSession(1l)); + assertNotNull(repo.getSession(2l)); + assertNotNull(repo.getSession(3l)); + clock.advance(Duration.ofSeconds(1)); + assertNotNull(repo.getSession(1l)); + assertNotNull(repo.getSession(2l)); + assertNotNull(repo.getSession(3l)); + clock.advance(Duration.ofSeconds(1)); + addSession(4l, 6); + assertNotNull(repo.getSession(1l)); + assertNotNull(repo.getSession(2l)); + assertNotNull(repo.getSession(3l)); + assertNotNull(repo.getSession(4l)); + clock.advance(Duration.ofSeconds(1)); + addSession(5l, 10); + assertNull(repo.getSession(1l)); + assertNull(repo.getSession(2l)); + assertNull(repo.getSession(3l)); + } + + @Test + public void require_that_all_sessions_are_deleted() { + repo.deleteAllSessions(); + assertNull(repo.getSession(1l)); + assertNull(repo.getSession(2l)); + assertNull(repo.getSession(3l)); + } + + private void addSession(long sessionId, long createTime) { + repo.addSession(new SessionHandlerTest.MockSession(sessionId, FilesApplicationPackage.fromFile(testApp), createTime)); + } + + @Test + public void require_that_sessions_belong_to_a_tenant() { + // tenant is "default" + assertNotNull(repo.getSession(1l)); + assertNotNull(repo.getSession(2l)); + assertNotNull(repo.getSession(3l)); + assertNull(repo.getSession(4l)); + + // tenant is "newTenant" + try { + setupSessions(TenantName.from("newTenant"), false); + } catch (Exception e) { + fail(); + } + assertNull(repo.getSession(1l)); + + repo.addSession(new SessionHandlerTest.MockSession(1l, FilesApplicationPackage.fromFile(testApp))); + repo.addSession(new SessionHandlerTest.MockSession(2l, FilesApplicationPackage.fromFile(testApp))); + assertNotNull(repo.getSession(1l)); + assertNotNull(repo.getSession(2l)); + assertNull(repo.getSession(3l)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java new file mode 100644 index 00000000000..4f638d54d46 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -0,0 +1,180 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.io.Files; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.provision.*; +import com.yahoo.path.Path; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.*; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class LocalSessionTest { + + private Path tenantPath = Path.createRoot(); + private Curator curator; + private ConfigCurator configCurator; + private TenantFileSystemDirs tenantFileSystemDirs; + private SuperModelGenerationCounter superModelGenerationCounter; + + @Before + public void setupTest() throws Exception { + curator = new MockCurator(); + configCurator = ConfigCurator.create(curator); + superModelGenerationCounter = new SuperModelGenerationCounter(curator); + tenantFileSystemDirs = new TenantFileSystemDirs(Files.createTempDir(), TenantName.from("test_tenant")); + } + + @Test + public void require_that_session_is_initialized() throws Exception { + LocalSession session = createSession(TenantName.defaultName(), 2); + assertThat(session.getSessionId(), is(2l)); + session = createSession(TenantName.defaultName(), Long.MAX_VALUE); + assertThat(session.getSessionId(), is(Long.MAX_VALUE)); + assertThat(session.getActiveSessionAtCreate(), is(0l)); + } + + @Test + public void require_that_session_status_is_updated() throws Exception { + LocalSession session = createSession(TenantName.defaultName(), 3); + assertThat(session.getStatus(), is(Session.Status.NEW)); + doPrepare(session); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + session.createActivateTransaction().commit(); + assertThat(session.getStatus(), is(Session.Status.ACTIVATE)); + } + + @Test + public void require_that_marking_session_modified_changes_status_to_new() throws Exception { + LocalSession session = createSession(TenantName.defaultName(), 3); + doPrepare(session); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + session.getApplicationFile(Path.createRoot(), LocalSession.Mode.READ); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + session.getApplicationFile(Path.createRoot(), LocalSession.Mode.WRITE); + assertThat(session.getStatus(), is(Session.Status.NEW)); + } + + @Test + public void require_that_preparer_is_run() throws Exception { + SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); + LocalSession session = createSession(TenantName.defaultName(), 3, preparer); + assertFalse(preparer.isPrepared); + doPrepare(session); + assertTrue(preparer.isPrepared); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + } + + @Test + public void require_that_session_status_can_be_deactivated() throws Exception { + SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); + LocalSession session = createSession(TenantName.defaultName(), 3, preparer); + session.createDeactivateTransaction().commit(); + assertThat(session.getStatus(), is(Session.Status.DEACTIVATE)); + } + + private File testApp = new File("src/test/apps/app"); + + @Test + public void require_that_application_file_can_be_fetched() throws Exception { + LocalSession session = createSession(TenantName.defaultName(), 3); + ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), LocalSession.Mode.READ); + ApplicationFile f2 = session.getApplicationFile(Path.fromString("services2.xml"), LocalSession.Mode.READ); + assertTrue(f1.exists()); + assertFalse(f2.exists()); + } + + @Test + public void require_that_session_can_be_deleted() throws Exception { + LocalSession session = createSession(TenantName.defaultName(), 3); + assertTrue(configCurator.exists("/3")); + assertTrue(new File(tenantFileSystemDirs.path(), "3").exists()); + long gen = superModelGenerationCounter.get(); + session.delete(); + assertThat(superModelGenerationCounter.get(), is(gen + 1)); + assertFalse(configCurator.exists("/3")); + assertFalse(new File(tenantFileSystemDirs.path(), "3").exists()); + } + + @Test(expected = IllegalStateException.class) + public void require_that_no_provision_info_throws_exception() throws Exception { + createSession(TenantName.defaultName(), 3).getProvisionInfo(); + } + + @Test + public void require_that_provision_info_can_be_read() throws Exception { + ProvisionInfo input = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("myhost", Collections.<String>emptyList()))); + + LocalSession session = createSession(TenantName.defaultName(), 3, new SessionTest.MockSessionPreparer(), Optional.of(input)); + ApplicationId origId = new ApplicationId.Builder() + .tenant("tenant") + .applicationName("foo").instanceName("quux").build(); + doPrepare(session, new PrepareParams().applicationId(origId)); + ProvisionInfo info = session.getProvisionInfo(); + assertNotNull(info); + assertThat(info.getHosts().size(), is(1)); + assertTrue(info.getHosts().contains(new HostSpec("myhost", Collections.emptyList()))); + } + + @Test + public void require_that_application_metadata_is_correct() throws Exception { + LocalSession session = createSession(TenantName.defaultName(), 3); + doPrepare(session, new PrepareParams()); + assertThat(session.getMetaData().toString(), is("n/a, n/a, 0, 0, , 0")); + } + + private LocalSession createSession(TenantName tenant, long sessionId) throws Exception { + SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); + return createSession(tenant, sessionId, preparer); + } + + private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer) throws Exception { + return createSession(tenant, sessionId, preparer, Optional.<ProvisionInfo>empty()); + } + + private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<ProvisionInfo> provisionInfo) throws Exception { + Path appPath = Path.fromString("/" + sessionId); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, appPath, provisionInfo); + zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); + ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, appPath); + if (provisionInfo.isPresent()) { + zkClient.feedProvisionInfos(Collections.singletonMap(Version.fromIntValues(0, 0, 0), provisionInfo.get())); + } + zkClient.feedZKFileRegistries(Collections.singletonMap(Version.fromIntValues(0, 0, 0), new MockFileRegistry())); + File sessionDir = new File(tenantFileSystemDirs.path(), String.valueOf(sessionId)); + sessionDir.createNewFile(); + return new LocalSession(tenant, sessionId, preparer, new SessionContext(FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, new MemoryApplicationRepo(), new HostRegistry<>(), superModelGenerationCounter)); + } + + private void doPrepare(LocalSession session) { + doPrepare(session, new PrepareParams()); + } + + private void doPrepare(LocalSession session, PrepareParams params) { + session.prepare(getLogger(false), params, Optional.empty(), tenantPath); + } + + DeployHandlerLogger getLogger(boolean verbose) { + return new DeployHandlerLogger(new Slime().get(), verbose, + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java new file mode 100644 index 00000000000..0af74cc9312 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; +import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.io.File; + +/** +* @author lulf +* @since 5.1 +*/ +public class MockFileDistributionFactory extends FileDistributionFactory { + + public final MockFileDistributionProvider mockFileDistributionProvider = new MockFileDistributionProvider(); + + public MockFileDistributionFactory() { + super(new MockCurator(), ""); + } + + @Override + public FileDistributionProvider createProvider(File applicationFile, ApplicationId applicationId) { + return mockFileDistributionProvider; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java new file mode 100644 index 00000000000..829c3f9008b --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.transaction.Transaction; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.util.Optional; + +/** + * Overrides application package fetching, because this part is hard to do without feeding a full app. + * + * @author lulf + * @since 5.1 + */ +public class MockSessionZKClient extends SessionZooKeeperClient { + + private ApplicationPackage app = null; + private Optional<ProvisionInfo> info = null; + private Session.Status sessionStatus; + + public MockSessionZKClient(Curator curator, Path rootPath) { + this(curator, rootPath, (ApplicationPackage)null); + } + + public MockSessionZKClient(Curator curator, Path rootPath, Optional<ProvisionInfo> provisionInfo) { + this(curator, rootPath); + this.info = provisionInfo; + } + + public MockSessionZKClient(Curator curator, Path rootPath, ApplicationPackage application) { + super(curator, rootPath); + this.app = application; + } + + public MockSessionZKClient(ApplicationPackage app) { + super(new MockCurator(), Path.createRoot()); + this.app = app; + } + + @Override + public ApplicationPackage loadApplicationPackage() { + if (app != null) return app; + return new MockApplicationPackage.Builder().withEmptyServices().build(); + } + + @Override + ProvisionInfo getProvisionInfo() { + return info.orElseThrow(() -> new IllegalStateException("Trying to read provision info, but no provision info exists")); + } + + @Override + public Transaction createWriteStatusTransaction(Session.Status status) { + return new DummyTransaction().add((DummyTransaction.RunnableOperation) () -> { + sessionStatus = status; + }); + } + + @Override + public Session.Status readStatus() { + return sessionStatus; + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java new file mode 100644 index 00000000000..9faba599e3a --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.container.jdisc.HttpRequest; + +import org.junit.Test; + +import java.util.Optional; +import java.util.Set; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author musum + */ +public class PrepareParamsTest { + + @Test + public void testCorrectParsing() { + PrepareParams prepareParams = createParams("http://foo:19071/application/v2/", + TenantName.defaultName()); + + assertThat(prepareParams.getApplicationId(), is(ApplicationId.defaultId())); + assertFalse(prepareParams.isDryRun()); + assertFalse(prepareParams.ignoreValidationErrors()); + assertThat(prepareParams.getVespaVersion(), is(Optional.<String>empty())); + assertTrue(prepareParams.getTimeoutBudget().hasTimeLeft()); + assertThat(prepareParams.getRotations().size(), is(0)); + } + + + static final String rotation = "rotation-042.vespa.a02.yahoodns.net"; + static final String vespaVersion = "6.37.49"; + static final String request = "http://foo:19071/application/v2/tenant/foo/application/bar?" + + PrepareParams.DRY_RUN_PARAM_NAME + "=true&" + + PrepareParams.IGNORE_VALIDATION_PARAM_NAME + "=false&" + + PrepareParams.APPLICATION_NAME_PARAM_NAME + "=baz&" + + PrepareParams.VESPA_VERSION_PARAM_NAME + "=" + vespaVersion + "&" + + PrepareParams.DOCKER_VESPA_IMAGE_VERSION_PARAM_NAME+ "=" + vespaVersion; + + @Test + public void testCorrectParsingWithRotation() { + PrepareParams prepareParams = createParams(request + "&" + + PrepareParams.ROTATIONS_PARAM_NAME + "=" + rotation, + TenantName.from("foo")); + + assertThat(prepareParams.getApplicationId().serializedForm(), is("foo:baz:default")); + assertTrue(prepareParams.isDryRun()); + assertFalse(prepareParams.ignoreValidationErrors()); + final Version expectedVersion = Version.fromString(vespaVersion); + assertThat(prepareParams.getVespaVersion().get(), is(expectedVersion)); + assertTrue(prepareParams.getTimeoutBudget().hasTimeLeft()); + final Set<Rotation> rotations = prepareParams.getRotations(); + assertThat(rotations.size(), is(1)); + assertThat(rotations, contains(equalTo(new Rotation(rotation)))); + assertThat(prepareParams.getDockerVespaImageVersion().get(), is(expectedVersion)); + } + + @Test + public void testCorrectParsingWithSeveralRotations() { + final String rotationTwo = "rotation-043.vespa.a02.yahoodns.net"; + final String twoRotations = rotation + "," + rotationTwo; + PrepareParams prepareParams = createParams(request + "&" + + PrepareParams.ROTATIONS_PARAM_NAME + "=" + twoRotations, + TenantName.from("foo")); + final Set<Rotation> rotations = prepareParams.getRotations(); + assertThat(rotations, containsInAnyOrder(new Rotation(rotation), new Rotation(rotationTwo))); + } + + // Create PrepareParams from a request (based on uri and tenant name) + private static PrepareParams createParams(String uri, TenantName tenantName) { + return PrepareParams.fromHttpRequest( + HttpRequest.createTestRequest(uri, + com.yahoo.jdisc.http.HttpRequest.Method.PUT), + tenantName, + new ConfigserverConfig(new ConfigserverConfig.Builder())); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java new file mode 100644 index 00000000000..6d8f93f4f8f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.*; + +import com.yahoo.vespa.config.server.application.ApplicationRepo; +import com.yahoo.vespa.curator.Curator; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.LongPredicate; + +/** + * @author lulf + * @since 5.1 + */ +public class RemoteSessionRepoTest extends TestWithCurator { + + private RemoteSessionRepo remoteSessionRepo; + + @Before + public void setupFacade() throws Exception { + createSession(2l, false); + createSession(3l, false); + curator.create(Path.fromString("/applications")); + curator.create(Path.fromString("/sessions")); + Tenant tenant = TenantBuilder.create(new TestComponentRegistry(curator), TenantName.defaultName(), Path.createRoot()).build(); + this.remoteSessionRepo = tenant.getRemoteSessionRepo(); + } + + private void createSession(long sessionId, boolean wait) { + createSession("", sessionId, wait); + } + + + private void createSession(String root, long sessionId, boolean wait) { + Path rootPath = Path.fromString(root).append("sessions"); + curator.create(rootPath); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, rootPath.append(String.valueOf(sessionId))); + zkc.createNewSession(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + if (wait) { + Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); + waiter.awaitCompletion(Duration.ofSeconds(120)); + } + } + + @Test + public void testInitialize() { + assertSessionExists(2l); + assertSessionExists(3l); + } + + @Test + public void testCreateSession() throws Exception { + createSession(0l, true); + assertSessionExists(0l); + } + + @Test + public void testSessionStateChange() throws Exception { + Path session = Path.fromString("/sessions/0"); + createSession(0l, true); + assertSessionStatus(0l, Session.Status.NEW); + assertStatusChange(0l, Session.Status.PREPARE); + assertStatusChange(0l, Session.Status.ACTIVATE); + + curator.delete(session); + assertSessionRemoved(0l); + assertNull(remoteSessionRepo.getSession(0l)); + } + + @Test + public void testBadApplicationRepoOnActivate() throws Exception { + ApplicationRepo applicationRepo = new FailingApplicationRepo(); + curator.framework().create().forPath("/mytenant"); + Tenant tenant = TenantBuilder.create(new TestComponentRegistry(curator), TenantName.from("mytenant"), Path.fromString("mytenant")) + .withApplicationRepo(applicationRepo) + .build(); + remoteSessionRepo = tenant.getRemoteSessionRepo(); + createSession("/mytenant", 2l, true); + assertThat(remoteSessionRepo.listSessions().size(), is(1)); + } + + private void assertStatusChange(long sessionId, Session.Status status) throws Exception { + Path statePath = Path.fromString("/sessions/" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + curator.create(statePath); + curatorFramework.setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); + System.out.println("Setting status " + status + " for " + sessionId); + assertSessionStatus(0l, status); + } + + private void assertSessionRemoved(long sessionId) { + waitFor(p -> remoteSessionRepo.getSession(sessionId) == null, sessionId); + assertNull(remoteSessionRepo.getSession(sessionId)); + } + + private void assertSessionExists(long sessionId) { + assertSessionStatus(sessionId, Session.Status.NEW); + } + + private void assertSessionStatus(long sessionId, Session.Status status) { + waitFor(p -> remoteSessionRepo.getSession(sessionId) != null && + remoteSessionRepo.getSession(sessionId).getStatus() == status, sessionId); + assertNotNull(remoteSessionRepo.getSession(sessionId)); + assertThat(remoteSessionRepo.getSession(sessionId).getStatus(), is(status)); + } + + private void waitFor(LongPredicate predicate, long sessionId) { + long endTime = System.currentTimeMillis() + 60_000; + boolean ok; + do { + ok = predicate.test(sessionId); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } while (System.currentTimeMillis() < endTime && !ok); + } + + private class FailingApplicationRepo implements ApplicationRepo { + @Override + public List<ApplicationId> listApplications() { + return Collections.singletonList(ApplicationId.defaultId()); + } + + @Override + public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { + return null; + } + + @Override + public long getSessionIdForApplication(ApplicationId applicationId) { + throw new IllegalArgumentException("Bad id " + applicationId); + } + + @Override + public void deleteApplication(ApplicationId applicationId) { + + } + + @Override + public void close() { + + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java new file mode 100644 index 00000000000..2fc65eb77a8 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -0,0 +1,284 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.io.Files; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.*; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Version; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.PathProvider; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.VespaModelFactory; + +import org.junit.Before; +import org.junit.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class RemoteSessionTest { + + private Curator curator; + private PathProvider pathProvider; + + @Before + public void setupTest() throws Exception { + curator = new MockCurator(); + pathProvider = new PathProvider(Path.createRoot()); + } + + @Test + public void require_that_session_is_initialized() { + Session session = createSession(2); + assertThat(session.getSessionId(), is(2l)); + session = createSession(Long.MAX_VALUE); + assertThat(session.getSessionId(), is(Long.MAX_VALUE)); + } + + @Test + public void require_that_applications_are_loaded() throws IOException, SAXException { + RemoteSession session = createSession(3, Arrays.asList(new MockModelFactory(), new VespaModelFactory(new NullConfigModelRegistry()))); + session.loadPrepared(); + ApplicationSet applicationSet = session.ensureApplicationLoaded(); + assertNotNull(applicationSet); + assertThat(applicationSet.getApplicationGeneration(), is(3l)); + assertThat(applicationSet.getForVersionOrLatest(Optional.empty()).getName(), is("foo")); + assertNotNull(applicationSet.getForVersionOrLatest(Optional.empty()).getModel()); + session.deactivate(); + + applicationSet = session.ensureApplicationLoaded(); + assertNotNull(applicationSet); + assertThat(applicationSet.getApplicationGeneration(), is(3l)); + assertThat(applicationSet.getForVersionOrLatest(Optional.empty()).getName(), is("foo")); + assertNotNull(applicationSet.getForVersionOrLatest(Optional.empty()).getModel()); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_new_invalid_application_throws_exception() throws IOException, SAXException { + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = Version.fromIntValues(1, 2, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = new MockModelFactory(); + okFactory.vespaVersion = Version.fromIntValues(1, 1, 0); + okFactory.throwOnLoad = false; + + RemoteSession session = createSession(3, Arrays.asList(okFactory, failingFactory)); + session.loadPrepared(); + } + + @Test + public void require_that_application_incompatible_with_latestmajor_is_loaded_on_earlier_major() throws IOException, SAXException { + MockModelFactory okFactory1 = new MockModelFactory(); + okFactory1.vespaVersion = Version.fromIntValues(1, 1, 0); + okFactory1.throwOnLoad = false; + + MockModelFactory okFactory2 = new MockModelFactory(); + okFactory2.vespaVersion = Version.fromIntValues(1, 2, 0); + okFactory2.throwOnLoad = false; + + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = Version.fromIntValues(2, 0, 0); + failingFactory.throwOnLoad = true; + + RemoteSession session = createSession(3, Arrays.asList(okFactory1, failingFactory, okFactory2)); + session.loadPrepared(); + } + + @Test + public void require_that_old_invalid_application_does_not_throw_exception_if_skipped() throws IOException, SAXException { + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = Version.fromIntValues(1, 1, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = + new MockModelFactory("<validation-overrides><allow until='2000-01-30'>skip-old-config-models</allow></validation-overrides>"); + okFactory.vespaVersion = Version.fromIntValues(1, 2, 0); + okFactory.throwOnLoad = false; + + RemoteSession session = createSession(3, Arrays.asList(okFactory, failingFactory)); + session.loadPrepared(); + } + + @Test + public void require_that_old_invalid_application_does_not_throw_exception_if_skipped_also_across_major_versions() throws IOException, SAXException { + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = Version.fromIntValues(1, 0, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = + new MockModelFactory("<validation-overrides><allow until='2000-01-30'>skip-old-config-models</allow></validation-overrides>"); + okFactory.vespaVersion = Version.fromIntValues(2, 0, 0); + okFactory.throwOnLoad = false; + + RemoteSession session = createSession(3, Arrays.asList(okFactory, failingFactory)); + session.loadPrepared(); + } + + @Test + public void require_that_old_invalid_application_does_not_throw_exception_if_skipped_also_when_new_major_is_incompatible() throws IOException, SAXException { + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = Version.fromIntValues(1, 0, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = + new MockModelFactory("<validation-overrides><allow until='2000-01-30'>skip-old-config-models</allow></validation-overrides>"); + okFactory.vespaVersion = Version.fromIntValues(1, 1, 0); + okFactory.throwOnLoad = false; + + MockModelFactory tooNewFactory = + new MockModelFactory("<validation-overrides><allow until='2000-01-30'>skip-old-config-models</allow></validation-overrides>"); + tooNewFactory.vespaVersion = Version.fromIntValues(2, 0, 0); + tooNewFactory.throwOnLoad = true; + + RemoteSession session = createSession(3, Arrays.asList(tooNewFactory, okFactory, failingFactory)); + session.loadPrepared(); + } + + @Test + public void require_that_an_application_package_can_limit_to_one_major_version() throws IOException, SAXException { + ApplicationPackage application = + new MockApplicationPackage.Builder().withServices("<services major-version='2' version=\"1.0\"></services>").build(); + + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = Version.fromIntValues(3, 0, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = new MockModelFactory(); + okFactory.vespaVersion = Version.fromIntValues(2, 0, 0); + okFactory.throwOnLoad = false; + + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(3), application); + RemoteSession session = createSession(3, zkc, Arrays.asList(okFactory, failingFactory)); + session.loadPrepared(); + + // Does not cause an exception because model version 3 is skipped + } + + @Test + public void require_that_session_status_is_updated() throws IOException, SAXException { + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(3)); + RemoteSession session = createSession(3, zkc); + assertThat(session.getStatus(), is(Session.Status.NEW)); + zkc.writeStatus(Session.Status.PREPARE); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + } + + @Test + public void require_that_permanent_app_is_used() { + Optional<PermanentApplicationPackage> permanentApp = Optional.of(new PermanentApplicationPackage( + new ConfigserverConfig(new ConfigserverConfig.Builder().applicationDirectory(Files.createTempDir().getAbsolutePath())))); + MockModelFactory mockModelFactory = new MockModelFactory(); + try { + int sessionId = 3; + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(sessionId)); + createSession(sessionId, zkc, Collections.singletonList(mockModelFactory), permanentApp).ensureApplicationLoaded(); + } catch (Exception e) { + e.printStackTrace(); + // ignore, we're not interested in deploy errors as long as the below state is OK. + } + assertNotNull(mockModelFactory.modelContext); + assertTrue(mockModelFactory.modelContext.permanentApplicationPackage().isPresent()); + } + + private RemoteSession createSession(long sessionId) { + return createSession(sessionId, Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); + } + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc) { + return createSession(sessionId, zkc, Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry()))); + } + private RemoteSession createSession(long sessionId, List<ModelFactory> modelFactories) { + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, pathProvider.getSessionDir(sessionId)); + return createSession(sessionId, zkc, modelFactories); + } + + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc, List<ModelFactory> modelFactories) { + return createSession(sessionId, zkc, modelFactories, Optional.empty()); + } + + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc, List<ModelFactory> modelFactories, Optional<PermanentApplicationPackage> permanentApplicationPackage) { + zkc.writeStatus(Session.Status.NEW); + zkc.writeApplicationId(new ApplicationId.Builder().applicationName("foo").instanceName("bim").build()); + return new RemoteSession(TenantName.from("default"), sessionId, new TestComponentRegistry(curator, new ModelFactoryRegistry(modelFactories), permanentApplicationPackage), zkc); + } + + private class MockModelFactory implements ModelFactory { + + public boolean throwOnLoad = false; + public ModelContext modelContext; + public Version vespaVersion = Version.fromIntValues(1, 2, 3); + + /** The validation overrides of this, or null if none */ + private final String validationOverrides; + + public MockModelFactory() { this(null); } + + public MockModelFactory(String validationOverrides) { + this.validationOverrides = validationOverrides; + } + + @Override + public Version getVersion() { + return vespaVersion; + } + + @Override + public Model createModel(ModelContext modelContext) { + if (throwOnLoad) { + throw new IllegalArgumentException("Foo"); + } + this.modelContext = modelContext; + return loadModel(); + } + + public Model loadModel() { + try { + Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); + ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().withValidationOverrides(validationOverrides).build(); + DeployState deployState = new DeployState.Builder().applicationPackage(application).now(now).build(); + return new VespaModel(deployState); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + if (throwOnLoad) { + throw new IllegalArgumentException("Foo"); + } + this.modelContext = modelContext; + return new ModelCreateResult(loadModel(), new ArrayList<>()); + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java new file mode 100644 index 00000000000..83ed65f03e4 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionFactoryTest.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.io.Files; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.http.SessionCreate; + +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class SessionFactoryTest extends TestWithTenant { + private SessionFactory factory; + + @Before + public void setup_test() throws Exception { + factory = tenant.getSessionFactory(); + } + + @Test + public void require_that_session_can_be_created() throws IOException { + LocalSession session = getLocalSession(); + assertNotNull(session); + assertThat(session.getSessionId(), is(2l)); + assertTrue(session.getCreateTime() > 0); + } + + @Test + public void require_that_application_name_is_set_in_application_package() throws IOException, JSONException { + LocalSession session = getLocalSession("book"); + assertNotNull(session); + ApplicationFile meta = session.getApplicationFile(Path.createRoot().append(".applicationMetaData"), LocalSession.Mode.READ); + assertTrue(meta.exists()); + JSONObject json = new JSONObject(IOUtils.readAll(meta.createReader())); + assertThat(json.getJSONObject("application").getString("name"), is("book")); + } + + @Test + public void require_that_session_can_be_created_from_existing() throws IOException { + LocalSession session = getLocalSession(); + assertNotNull(session); + assertThat(session.getSessionId(), is(2l)); + LocalSession session2 = factory.createSessionFromExisting(session, new BaseDeployLogger(), TimeoutBudgetTest.day()); + assertNotNull(session2); + assertThat(session2.getSessionId(), is(3l)); + } + + @Test(expected = RuntimeException.class) + public void require_that_invalid_app_dir_is_handled() throws IOException { + factory.createSession(new File("doesnotpointtoavaliddir"), "music", new BaseDeployLogger(), TimeoutBudgetTest.day()); + } + + private LocalSession getLocalSession() throws IOException { + return getLocalSession("music"); + } + + private LocalSession getLocalSession(String appName) throws IOException { + CompressedApplicationInputStream app = CompressedApplicationInputStream.createFromCompressedStream(new FileInputStream(CompressedApplicationInputStreamTest.createTarFile()), SessionCreate.APPLICATION_X_GZIP); + return factory.createSession(app.decompress(Files.createTempDir()), appName, new BaseDeployLogger(), TimeoutBudgetTest.day()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java new file mode 100644 index 00000000000..308e6aa29d2 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -0,0 +1,286 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.application.MemoryApplicationRepo; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.configchange.MockRestartAction; +import com.yahoo.vespa.config.server.configchange.RestartActions; +import com.yahoo.vespa.config.server.http.InvalidApplicationException; +import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.*; + +/** + * @author lulf + * @since 5.1 + */ +public class SessionPreparerTest extends TestWithCurator { + + private static final Path appPath = Path.createRoot().append("testapp"); + private static final File testApp = new File("src/test/apps/app"); + private static final File invalidTestApp = new File("src/test/apps/illegalApp"); + + private SessionPreparer preparer; + private TestComponentRegistry componentRegistry; + private MockFileDistributionFactory fileDistributionFactory; + private Path tenantPath = appPath; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void setUp() throws Exception { + componentRegistry = new TestComponentRegistry(curator); + fileDistributionFactory = (MockFileDistributionFactory)componentRegistry.getFileDistributionFactory(); + preparer = createPreparer(); + } + + private SessionPreparer createPreparer() { + return createPreparer(HostProvisionerProvider.empty()); + } + + private SessionPreparer createPreparer(HostProvisionerProvider hostProvisionerProvider) { + ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Arrays.asList( + new TestModelFactory(Version.fromIntValues(1, 2, 3)), + new TestModelFactory(Version.fromIntValues(3, 2, 1)))); + return createPreparer(modelFactoryRegistry, hostProvisionerProvider); + } + + private SessionPreparer createPreparer(ModelFactoryRegistry modelFactoryRegistry, + HostProvisionerProvider hostProvisionerProvider) { + return new SessionPreparer( + modelFactoryRegistry, + componentRegistry.getFileDistributionFactory(), + hostProvisionerProvider, + new PermanentApplicationPackage(componentRegistry.getConfigserverConfig()), + componentRegistry.getConfigserverConfig(), + componentRegistry.getConfigDefinitionRepo(), + curator, + componentRegistry.getZone()); + } + + @Test(expected = InvalidApplicationException.class) + public void require_that_application_validation_exception_is_not_caught() throws IOException, SAXException { + FilesApplicationPackage app = getApplicationPackage(invalidTestApp); + preparer.prepare(getContext(app), getLogger(), new PrepareParams(), Optional.empty(), tenantPath); + } + + @Test + public void require_that_application_validation_exception_is_ignored_if_forced() throws IOException, SAXException { + FilesApplicationPackage app = getApplicationPackage(invalidTestApp); + preparer.prepare(getContext(app), getLogger(), new PrepareParams().ignoreValidationErrors(true).timeoutBudget(TimeoutBudgetTest.day()), Optional.empty(), tenantPath); + } + + @Test + public void require_that_zookeeper_is_not_written_to_if_dryrun() throws IOException { + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()), Optional.empty(), tenantPath); + assertFalse(configCurator.exists(appPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); + } + + @Test + public void require_that_filedistribution_is_ignored_on_dryrun() throws IOException { + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams().dryRun(true).timeoutBudget(TimeoutBudgetTest.day()), Optional.empty(), tenantPath); + assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(0)); + assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().limitSendingOfDeployedFilesToCalled, is(0)); + assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(0)); + } + + @Test + public void require_that_application_is_prepared() throws Exception { + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams(), Optional.empty(), tenantPath); + assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().sendDeployedFilesCalled, is(2)); + assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().limitSendingOfDeployedFilesToCalled, is(2)); + // Should be called only once no matter how many model versions are built + assertThat(fileDistributionFactory.mockFileDistributionProvider.getMockFileDBHandler().reloadDeployFileDistributorCalled, is(1)); + assertTrue(configCurator.exists(appPath.append(ConfigCurator.USERAPP_ZK_SUBPATH).append("services.xml").getAbsolute())); + } + + @Test + public void require_that_prepare_succeeds_if_newer_version_fails() throws IOException { + ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Arrays.asList( + new TestModelFactory(Version.fromIntValues(1, 2, 3)), + new FailingModelFactory(Version.fromIntValues(3, 2, 1), new IllegalArgumentException("BOOHOO")))); + preparer = createPreparer(modelFactoryRegistry, HostProvisionerProvider.empty()); + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams(), Optional.empty(), tenantPath); + } + + @Test(expected = InvalidApplicationException.class) + public void require_that_prepare_fails_if_older_version_fails() throws IOException { + ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Arrays.asList( + new TestModelFactory(Version.fromIntValues(3, 2, 3)), + new FailingModelFactory(Version.fromIntValues(1, 2, 1), new IllegalArgumentException("BOOHOO")))); + preparer = createPreparer(modelFactoryRegistry, HostProvisionerProvider.empty()); + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams(), Optional.empty(), tenantPath); + } + + @Test(expected = InvalidApplicationException.class) + public void require_exception_for_overlapping_host() throws IOException { + SessionContext ctx = getContext(getApplicationPackage(testApp)); + ((HostRegistry<ApplicationId>)ctx.getHostValidator()).update(applicationId("foo"), Collections.singletonList("mytesthost")); + preparer.prepare(ctx, new BaseDeployLogger(), new PrepareParams(), Optional.empty(), tenantPath); + } + + @Test + public void require_no_warning_for_overlapping_host_for_same_appid() throws IOException { + SessionContext ctx = getContext(getApplicationPackage(testApp)); + ((HostRegistry<ApplicationId>)ctx.getHostValidator()).update(applicationId("default"), Collections.singletonList("mytesthost")); + final StringBuilder logged = new StringBuilder(); + DeployLogger logger = (level, message) -> { + System.out.println(level + ": "+message); + if (level.equals(LogLevel.WARNING) && message.contains("The host mytesthost is already in use")) logged.append("ok"); + }; + preparer.prepare(ctx, logger, new PrepareParams(), Optional.empty(), tenantPath); + assertEquals(logged.toString(), ""); + } + + @Test + public void require_that_application_id_is_written_in_prepare() throws IOException { + TenantName tenant = TenantName.from("tenant"); + ApplicationId origId = new ApplicationId.Builder() + .tenant(tenant) + .applicationName("foo").instanceName("quux").build(); + PrepareParams params = new PrepareParams().applicationId(origId); + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), params, Optional.empty(), tenantPath); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, appPath); + assertTrue(configCurator.exists(appPath.append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute())); + assertThat(zkc.readApplicationId(tenant), is(origId)); + } + + @Test + public void require_that_config_change_actions_are_collected_from_all_models() throws IOException { + ServiceInfo service = new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName"); + ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Arrays.asList( + new ConfigChangeActionsModelFactory(Version.fromIntValues(1, 2, 3), + new MockRestartAction("change", Arrays.asList(service))), + new ConfigChangeActionsModelFactory(Version.fromIntValues(1, 2, 4), + new MockRestartAction("other change", Arrays.asList(service))))); + preparer = createPreparer(modelFactoryRegistry, HostProvisionerProvider.empty()); + List<RestartActions.Entry> actions = + preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), new PrepareParams(), Optional.empty(), tenantPath). + getRestartActions().getEntries(); + assertThat(actions.size(), is(1)); + assertThat(actions.get(0).getMessages(), equalTo(ImmutableSet.of("change", "other change"))); + } + + private Set<Rotation> readRotationsFromZK(ApplicationId applicationId) { + return new RotationsCache(curator, tenantPath).readRotationsFromZooKeeper(applicationId); + } + + @Test + public void require_that_rotations_are_written_in_prepare() throws IOException { + final String rotations = "mediasearch.msbe.global.vespa.yahooapis.com"; + final ApplicationId applicationId = applicationId("test"); + PrepareParams params = new PrepareParams().applicationId(applicationId).rotations(rotations); + File app = new File("src/test/resources/deploy/app"); + preparer.prepare(getContext(getApplicationPackage(app)), getLogger(), params, Optional.empty(), tenantPath); + assertThat(readRotationsFromZK(applicationId), contains(new Rotation(rotations))); + } + + @Test + public void require_that_rotations_are_read_from_zookeeper_and_used() throws IOException { + final Version vespaVersion = Version.fromIntValues(1, 2, 3); + final TestModelFactory modelFactory = new TestModelFactory(vespaVersion); + preparer = createPreparer(new ModelFactoryRegistry(Arrays.asList(modelFactory)), + HostProvisionerProvider.empty()); + + final String rotations = "foo.msbe.global.vespa.yahooapis.com"; + final ApplicationId applicationId = applicationId("test"); + new RotationsCache(curator, tenantPath).writeRotationsToZooKeeper(applicationId, Collections.singleton(new Rotation(rotations))); + final PrepareParams params = new PrepareParams().applicationId(applicationId); + final File app = new File("src/test/resources/deploy/app"); + preparer.prepare(getContext(getApplicationPackage(app)), getLogger(), params, Optional.empty(), tenantPath); + + // check that the rotation from zookeeper were used + final ModelContext modelContext = modelFactory.getModelContext(); + final Set<Rotation> rotationSet = modelContext.properties().rotations(); + assertThat(rotationSet, contains(new Rotation(rotations))); + + // Check that the persisted value is still the same + assertThat(readRotationsFromZK(applicationId), contains(new Rotation(rotations))); + } + + private SessionContext getContext(FilesApplicationPackage app) throws IOException { + return new SessionContext(app, new SessionZooKeeperClient(curator, appPath), app.getAppDir(), new MemoryApplicationRepo(), new HostRegistry<>(), new SuperModelGenerationCounter(curator)); + } + + private FilesApplicationPackage getApplicationPackage(File testFile) throws IOException { + File appDir = folder.newFolder(); + IOUtils.copyDirectory(testFile, appDir); + return FilesApplicationPackage.fromFile(appDir); + } + + DeployHandlerLogger getLogger() { + return getLogger(false); + } + + DeployHandlerLogger getLogger(boolean verbose) { + return new DeployHandlerLogger(new Slime().get(), verbose, + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + } + + private static class FailingModelFactory extends TestModelFactory { + private final RuntimeException exception; + public FailingModelFactory(Version vespaVersion, RuntimeException exception) { + super(vespaVersion); + this.exception = exception; + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + throw exception; + } + } + + private ApplicationId applicationId(String applicationName) { + return ApplicationId.from(TenantName.defaultName(), + ApplicationName.from(applicationName), InstanceName.defaultName()); + } + + private static class ConfigChangeActionsModelFactory extends TestModelFactory { + private final ConfigChangeAction action; + public ConfigChangeActionsModelFactory(Version vespaVersion, ConfigChangeAction action) { + super(vespaVersion); + this.action = action; + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + ModelCreateResult result = super.createAndValidateModel(modelContext, ignoreValidationErrors); + return new ModelCreateResult(result.getModel(), Arrays.asList(action)); + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java new file mode 100644 index 00000000000..8549902faf0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepoTest.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import org.junit.Test; + +import com.yahoo.config.provision.TenantName; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author musum + * @since 5.1.14 + */ +public class SessionRepoTest { + @Test + public void require_that_sessionrepo_is_initialized() { + SessionRepo<TestSession> sessionRepo = new SessionRepo<>(); + assertNull(sessionRepo.getSession(1L)); + sessionRepo.addSession(new TestSession(1)); + assertThat(sessionRepo.getSession(1L).getSessionId(), is(1l)); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_adding_existing_session_fails() { + SessionRepo<TestSession> sessionRepo = new SessionRepo<>(); + final TestSession session = new TestSession(1); + sessionRepo.addSession(session); + sessionRepo.addSession(session); + } + + private class TestSession extends Session { + public TestSession(long sessionId) { + super(TenantName.from("default"), sessionId); + } + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java new file mode 100644 index 00000000000..de64db1bbc9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationSet; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.util.ArrayList; +import java.util.Optional; + +/** + * @author lulf + * @since 5.1 + */ +public class SessionTest { + + public static class MockSessionPreparer extends SessionPreparer { + public boolean isPrepared = false; + + public MockSessionPreparer() { + super(null, null, null, null, null, null, new MockCurator(), null); + } + + @Override + public ConfigChangeActions prepare(SessionContext context, DeployLogger logger, PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath) { + isPrepared = true; + return new ConfigChangeActions(new ArrayList<>()); + } + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java new file mode 100644 index 00000000000..4508d8c234f --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.TestWithCurator; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class SessionZooKeeperClientTest extends TestWithCurator { + + @Test + public void require_that_status_can_be_updated() { + SessionZooKeeperClient zkc = createSessionZKClient("1"); + zkc.writeStatus(Session.Status.NEW); + assertThat(zkc.readStatus(), is(Session.Status.NEW)); + + zkc.writeStatus(Session.Status.PREPARE); + assertThat(zkc.readStatus(), is(Session.Status.PREPARE)); + + zkc.writeStatus(Session.Status.ACTIVATE); + assertThat(zkc.readStatus(), is(Session.Status.ACTIVATE)); + + zkc.writeStatus(Session.Status.DEACTIVATE); + assertThat(zkc.readStatus(), is(Session.Status.DEACTIVATE)); + } + + @Test + public void require_that_status_is_written_to_zk() { + SessionZooKeeperClient zkc = createSessionZKClient("2"); + zkc.writeStatus(Session.Status.NEW); + String path = "/2" + ConfigCurator.SESSIONSTATE_ZK_SUBPATH; + assertTrue(configCurator.exists(path)); + assertThat(configCurator.getData(path), is("NEW")); + } + + @Test + public void require_that_status_is_read_from_zk() { + SessionZooKeeperClient zkc = createSessionZKClient("3"); + curator.set(Path.fromString("3").append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH), Utf8.toBytes("PREPARE")); + assertThat(zkc.readStatus(), is(Session.Status.PREPARE)); + } + + @Test + public void require_that_application_id_is_written_to_zk() { + ApplicationId id = new ApplicationId.Builder() + .tenant("tenant") + .applicationName("foo").instanceName("bim").build(); + SessionZooKeeperClient zkc = createSessionZKClient("3"); + zkc.writeApplicationId(id); + String path = "/3/" + SessionZooKeeperClient.APPLICATION_ID_PATH; + assertTrue(configCurator.exists(path)); + assertThat(configCurator.getData(path), is("tenant:foo:bim")); + } + + @Test + public void require_that_application_id_is_read_from_zk() { + ApplicationId id = new ApplicationId.Builder() + .tenant("tenant") + .applicationName("bar").instanceName("quux").build(); + String idNoVersion = id.serializedForm(); + assertApplicationIdParse("3", idNoVersion, idNoVersion); + } + + @Test + public void require_that_default_name_is_returned_if_node_does_not_exist() { + assertThat(createSessionZKClient("3").readApplicationId(TenantName.defaultName()).application().value(), is("default")); + } + + @Test + public void require_that_create_time_can_be_written_and_read() { + SessionZooKeeperClient zkc = createSessionZKClient("3"); + curator.delete(Path.fromString("3")); + assertThat(zkc.readCreateTime(), is(0l)); + zkc.createNewSession(123456l, TimeUnit.SECONDS); + assertThat(zkc.readCreateTime(), is(123456l)); + } + + @Test + public void require_that_create_time_has_correct_unit() { + SessionZooKeeperClient zkc = createSessionZKClient("3"); + curator.delete(Path.fromString("3")); + assertThat(zkc.readCreateTime(), is(0l)); + zkc.createNewSession(60, TimeUnit.MINUTES); + assertThat(zkc.readCreateTime(), is(3600l)); + } + + private void assertApplicationIdParse(String sessionId, String idString, String expectedIdString) { + SessionZooKeeperClient zkc = createSessionZKClient(sessionId); + String path = "/" + sessionId + "/" + SessionZooKeeperClient.APPLICATION_ID_PATH; + configCurator.putData(path, idString); + ApplicationId zkId = zkc.readApplicationId(TenantName.defaultName()); + assertThat(zkId.serializedForm(), is(expectedIdString)); + } + + private SessionZooKeeperClient createSessionZKClient(String generation) { + return createSessionZKClient(generation, 100); + } + + private SessionZooKeeperClient createSessionZKClient(String generation, long createTimeInMillis) { + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, Path.fromString(generation)); + zkc.createNewSession(createTimeInMillis, TimeUnit.MILLISECONDS); + return zkc; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg new file mode 100644 index 00000000000..0bc17bae65e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg @@ -0,0 +1,18 @@ +asyncfetchocc 10 +e 4 +search[2].feeder[1] "bazfeeder" +search[1].feeder[0] "barfeeder1_1" +search[1].feeder[3] "barfeeder2_1" +onlyindef 45 + +speciallog[0].filehandler.rotation "0 1 ..." + +rulebase[4] +rulebase[0].name "cjk" +rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n" +rulebase[1].name "common" +rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" +rulebase[2].name "egyik" +rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" +rulebase[3].name "masik" +rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg new file mode 100644 index 00000000000..88b50384058 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg @@ -0,0 +1 @@ +usercfgwithid 86 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg new file mode 100644 index 00000000000..b34c4ed311e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg @@ -0,0 +1 @@ +foo "test" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg new file mode 100644 index 00000000000..12c5b53de7d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg @@ -0,0 +1 @@ +theint 34 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg new file mode 100644 index 00000000000..73ed41667f9 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg @@ -0,0 +1 @@ +keepsuccess true diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java new file mode 100644 index 00000000000..f517145fd91 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.version; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + */ +public class VersionStateTest { + + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); + + @Test + public void upgrade() throws IOException { + Version unknownVersion = Version.fromIntValues(0, 0, 0); + File versionFile = tempDir.newFile(); + VersionState state = new VersionState(versionFile); + assertThat(state.storedVersion(), is(unknownVersion)); + assertTrue(state.isUpgraded()); + state.saveNewVersion(); + assertFalse(state.isUpgraded()); + + IOUtils.writeFile(versionFile, "badversion", false); + assertThat(state.storedVersion(), is(unknownVersion)); + assertTrue(state.isUpgraded()); + + IOUtils.writeFile(versionFile, "5.0.0", false); + assertThat(state.storedVersion(), is(Version.fromIntValues(5, 0, 0))); + assertTrue(state.isUpgraded()); + + state.saveNewVersion(); + assertThat(state.currentVersion(), is(state.storedVersion())); + assertFalse(state.isUpgraded()); + } + + @Test + public void serverdbfile() throws IOException { + File dbDir = tempDir.newFolder(); + VersionState state = new VersionState(new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()))); + state.saveNewVersion(); + File versionFile = new File(dbDir, "vespa_version"); + assertTrue(versionFile.exists()); + Version stored = Version.fromString(IOUtils.readFile(versionFile)); + assertThat(stored, is(state.currentVersion())); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java new file mode 100644 index 00000000000..b370b148fe0 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ConfigCuratorTest.java @@ -0,0 +1,239 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Tests the ZKFacade using a curator mock. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class ConfigCuratorTest { + + private final String defKey1 = "attributes"; + + private final String payload1 = "attribute[5]\n" + + "attribute[0].name Popularity\n" + + "attribute[0].datatype string\n" + + "attribute[0].collectiontype single\n" + + "attribute[0].removeifzero false\n" + + "attribute[0].createifnonexistent false\n" + + "attribute[0].loadtype \"always\"\n" + + "attribute[0].uniqueonly false\n" + + "attribute[0].sparse false\n" + + "attribute[0].noupdate false\n" + + "attribute[0].fastsearch false\n" + + "attribute[0].fastaggregate false\n" + + "attribute[0].fastersearch false\n" + + "attribute[1].name atA\n" + + "attribute[1].datatype string\n" + + "attribute[1].collectiontype weightedset\n" + + "attribute[1].removeifzero false\n" + + "attribute[1].createifnonexistent false\n" + + "attribute[1].loadtype \"always\"\n" + + "attribute[1].uniqueonly false\n" + + "attribute[1].sparse false\n" + + "attribute[1].noupdate false\n" + + "attribute[1].fastsearch true\n" + + "attribute[1].fastaggregate false\n" + + "attribute[1].fastersearch false\n" + + "attribute[2].name default_fieldlength\n" + + "attribute[2].datatype uint32\n" + + "attribute[2].collectiontype single\n" + + "attribute[2].removeifzero false\n" + + "attribute[2].createifnonexistent false\n" + + "attribute[2].loadtype \"always\"\n" + + "attribute[2].uniqueonly false\n" + + "attribute[2].sparse false\n" + + "attribute[2].noupdate true\n" + + "attribute[2].fastsearch false\n" + + "attribute[2].fastaggregate false\n" + + "attribute[2].fastersearch false\n" + + "attribute[3].name default_literal_fieldlength\n" + + "attribute[3].datatype uint32\n" + + "attribute[3].collectiontype single\n" + + "attribute[3].removeifzero false\n" + + "attribute[3].createifnonexistent false\n" + + "attribute[3].loadtype \"always\"\n" + + "attribute[3].uniqueonly false\n" + + "attribute[3].sparse false\n" + + "attribute[3].noupdate true\n" + + "attribute[3].fastsearch false\n" + + "attribute[3].fastaggregate false\n" + + "attribute[3].fastersearch false\n" + + "attribute[4].name artist_fieldlength\n" + + "attribute[4].datatype uint32\n" + + "attribute[4].collectiontype single\n" + + "attribute[4].removeifzero false\n" + + "attribute[4].createifnonexistent false\n" + + "attribute[4].loadtype \"always\"\n" + + "attribute[4].uniqueonly false\n" + + "attribute[4].sparse false\n" + + "attribute[4].noupdate true\n" + + "attribute[4].fastsearch false\n" + + "attribute[4].fastaggregate false\n" + + "attribute[4].fastersearch false\n"; + + private final String payload3 = "attribute[5]\n" + + "attribute[0].name Popularity\n" + + "attribute[0].datatype String\n" + + "attribute[0].collectiontype single\n" + + "attribute[0].removeifzero false\n" + + "attribute[0].createifnonexistent false\n" + + "attribute[0].loadtype \"always\"\n" + + "attribute[0].uniqueonly false\n" + + "attribute[0].sparse false\n" + + "attribute[0].noupdate false\n" + + "attribute[0].fastsearch false\n" + + "attribute[0].fastaggregate false\n" + + "attribute[0].fastersearch false\n" + + "attribute[1].name atA\n" + + "attribute[1].datatype string\n" + + "attribute[1].collectiontype weightedset\n" + + "attribute[1].removeifzero false\n" + + "attribute[1].createifnonexistent false\n" + + "attribute[1].loadtype \"always\"\n" + + "attribute[1].uniqueonly false\n" + + "attribute[1].sparse false\n" + + "attribute[1].noupdate false\n" + + "attribute[1].fastsearch true\n" + + "attribute[1].fastaggregate false\n" + + "attribute[1].fastersearch false\n" + + "attribute[2].name default_fieldlength\n" + + "attribute[2].datatype uint32\n" + + "attribute[2].collectiontype single\n" + + "attribute[2].removeifzero false\n" + + "attribute[2].createifnonexistent false\n" + + "attribute[2].loadtype \"always\"\n" + + "attribute[2].uniqueonly false\n" + + "attribute[2].sparse false\n" + + "attribute[2].noupdate true\n" + + "attribute[2].fastsearch false\n" + + "attribute[2].fastaggregate false\n" + + "attribute[2].fastersearch false\n" + + "attribute[3].name default_literal_fieldlength\n" + + "attribute[3].datatype uint32\n" + + "attribute[3].collectiontype single\n" + + "attribute[3].removeifzero false\n" + + "attribute[3].createifnonexistent false\n" + + "attribute[3].loadtype \"always\"\n" + + "attribute[3].uniqueonly false\n" + + "attribute[3].sparse false\n" + + "attribute[3].noupdate true\n" + + "attribute[3].fastsearch false\n" + + "attribute[3].fastaggregate false\n" + + "attribute[3].fastersearch false\n" + + "attribute[4].name artist_fieldlength\n" + + "attribute[4].datatype uint32\n" + + "attribute[4].collectiontype single\n" + + "attribute[4].removeifzero false\n" + + "attribute[4].createifnonexistent false\n" + + "attribute[4].loadtype \"always\"\n" + + "attribute[4].uniqueonly false\n" + + "attribute[4].sparse false\n" + + "attribute[4].noupdate true\n" + + "attribute[4].fastsearch false\n" + + "attribute[4].fastaggregate false\n" + + "attribute[4].fastersearch false\n"; + + private void initAndClearZK(ConfigCurator zkIf) { + zkIf.initAndClear(ConfigCurator.DEFCONFIGS_ZK_SUBPATH); + zkIf.initAndClear(ConfigCurator.USERAPP_ZK_SUBPATH); + } + + private ConfigCurator deployApp() { + ConfigCurator zkIf = getFacade(); + initAndClearZK(zkIf); + zkIf.putData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, defKey1, payload1); + // zkIf.putData(ConfigCurator.USERCONFIGS_ZK_SUBPATH, cfgKey1, payload3); + String partitionsDef = "version=7\\n" + + "dataset[].id int\\n" + + "dataset[].partbits int default=6"; + zkIf.putData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, "partitions", partitionsDef); + String partitionsUser = "dataset[0].partbits 8\\n"; + // zkIf.putData(ConfigCurator.USERCONFIGS_ZK_SUBPATH, "partitions", partitionsUser); + return zkIf; + } + + @Test + public void testZKInterface() { + ConfigCurator zkIf = getFacade(); + zkIf.putData("", "test", "foo"); + zkIf.putData("/test", "me", "bar"); + zkIf.putData("", "test;me;now,then", "baz"); + assertEquals(zkIf.getData("", "test"), "foo"); + assertEquals(zkIf.getData("/test", "me"), "bar"); + assertEquals(zkIf.getData("", "test;me;now,then"), "baz"); + } + + @Test + public void testWatcher() { + ConfigCurator zkIf = getFacade(); + + String data = zkIf.getData("/nothere"); + assertNull(data); + zkIf.putData("", "/nothere", "foo"); + assertEquals(zkIf.getData("/nothere"), "foo"); + + zkIf.putData("", "test", "foo"); + data = zkIf.getData("/test"); + assertEquals(data, "foo"); + zkIf.putData("", "/test", "bar"); + data = zkIf.getData("/test"); + assertEquals(data, "bar"); + + zkIf.getChildren("/"); + zkIf.putData("", "test2", "foo2"); + } + + private ConfigCurator getFacade() { + return ConfigCurator.create(new MockCurator()); + } + + @Test + public void testGetDeployedData() { + ConfigCurator zkIf = deployApp(); + assertEquals(zkIf.getData(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, defKey1), payload1); + } + + @Test + public void testEmptyData() { + ConfigCurator zkIf = getFacade(); + zkIf.createNode("/empty", "data"); + assertEquals("", zkIf.getData("/empty", "data")); + } + + @Test + public void testRecursiveDelete() { + ConfigCurator configCurator = getFacade(); + configCurator.putData("/foo", Utf8.toBytes("sadsdfsdfsdfsdf")); + configCurator.putData("/foo/bar", Utf8.toBytes("dsfsdffds")); + configCurator.putData("/foo/baz", + Utf8.toBytes("sdf\u00F8l ksdfl skdflsk dflsdkfd welkr3k lkr e4kt4 54l4l353k l534klk3lk4l33k5l 353l4k l43k l4k")); + configCurator.putData("/foo/bar/dill", Utf8.toBytes("sdfsfe 23 42 3 3 2342")); + configCurator.putData("/foo", Utf8.toBytes("sdcfsdfsdf")); + configCurator.putData("/foo", Utf8.toBytes("sdcfsd sdfdffsdf")); + configCurator.deleteRecurse("/foo"); + assertFalse(configCurator.exists("/foo")); + assertFalse(configCurator.exists("/foo/bar")); + assertFalse(configCurator.exists("/foo/bar/dill")); + assertFalse(configCurator.exists("/foo/bar/baz")); + try { + configCurator.getChildren("/foo"); + fail("Got children from nonexisting ZK path"); + } catch (RuntimeException e) { + assertTrue(e.getCause().getMessage().matches(".*NoNode.*")); + } + configCurator.deleteRecurse("/nonexisting"); + } + + @Test + public void testGetZkNodePath() { + assertEquals("foo,1", ConfigCurator.getZkNodePath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, "foo", "1", "a/b")); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java new file mode 100644 index 00000000000..95d17156e63 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounterTest.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.yahoo.vespa.config.server.TestWithCurator; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author lulf + * @since 5.1 + */ +public class InitializedCounterTest extends TestWithCurator { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void setupZK() { + configCurator.createNode("/sessions"); + configCurator.createNode("/sessions/1"); + configCurator.createNode("/sessions/2"); + } + + @Test + public void requireThatCounterIsInitializedFromNumberOfSessions() { + InitializedCounter counter = new InitializedCounter(curator, "/counter", "/sessions"); + assertThat(counter.counter.get(), is(2l)); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java new file mode 100644 index 00000000000..6205ac09c4c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import com.google.common.io.Files; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationFileTest; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class ZKApplicationFileTest extends ApplicationFileTest { + + private void feed(ConfigCurator zk, File dirToFeed) throws IOException { + assertTrue(dirToFeed.isDirectory()); + String appPath = "/0"; + zk.feedZooKeeper(dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + zk.putData(appPath, ZKApplicationPackage.fileRegistryNode, "dummyfiles"); + } + + @Override + public ApplicationFile getApplicationFile(Path path) throws IOException{ + ConfigCurator configCurator = ConfigCurator.create(new MockCurator()); + File tmp = Files.createTempDir(); + writeAppTo(tmp); + feed(configCurator, tmp); + return new ZKApplicationFile(path, new ZKLiveApp(configCurator, Path.fromString("/0"))); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java new file mode 100644 index 00000000000..217e2a04f9b --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.zookeeper; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.util.Collections; +import java.util.regex.Pattern; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.Version; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.TestWithCurator; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.yahoo.io.IOUtils; + +public class ZKApplicationPackageTest extends TestWithCurator { + + private static final String APP = "src/test/apps/zkapp"; + private static final ProvisionInfo provisionInfo = ProvisionInfo.withHosts( + Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList()))); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void testBasicZKFeed() throws IOException { + feed(configCurator, new File(APP)); + ZKApplicationPackage zkApp = new ZKApplicationPackage(configCurator, Path.fromString("/0")); + assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getServices())).matches()); + assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); + assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); + DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); + assertEquals(deployState.getSearchDefinitions().size(), 5); + assertEquals(zkApp.searchDefinitionContents().size(), 5); + assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); + assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3); + assertEquals(zkApp.getFileReference(Path.fromString("components/file.txt")).getAbsolutePath(), "/home/vespa/test/file.txt"); + try (Reader foo = zkApp.getFile(Path.fromString("files/foo.json")).createReader()) { + assertEquals(IOUtils.readAll(foo), "foo : foo\n"); + } + try (Reader bar = zkApp.getFile(Path.fromString("files/sub/bar.json")).createReader()) { + assertEquals(IOUtils.readAll(bar), "bar : bar\n"); + } + assertTrue(zkApp.getFile(Path.createRoot()).exists()); + assertTrue(zkApp.getFile(Path.createRoot()).isDirectory()); + Version goodVersion = Version.fromIntValues(3, 0, 0); + assertTrue(zkApp.getFileRegistryMap().containsKey(goodVersion)); + assertFalse(zkApp.getFileRegistryMap().containsKey(Version.fromIntValues(0, 0, 0))); + assertThat(zkApp.getFileRegistryMap().get(goodVersion).fileSourceHost(), is("dummyfiles")); + assertTrue(zkApp.getProvisionInfoMap().containsKey(goodVersion)); + ProvisionInfo readInfo = zkApp.getProvisionInfoMap().get(goodVersion); + assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(provisionInfo.toJson()))); + assertTrue(zkApp.getDeployment().isPresent()); + assertThat(DeploymentSpec.fromXml(zkApp.getDeployment().get()).globalServiceId().get(), is("mydisc")); + } + + private void feed(ConfigCurator zk, File dirToFeed) throws IOException { + assertTrue(dirToFeed.isDirectory()); + zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; + zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); + zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); + zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode + "/3.0.0", provisionInfo.toJson()); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/ServiceModelTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/ServiceModelTest.java new file mode 100644 index 00000000000..cbaa8f36805 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/serviceview/ServiceModelTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import static org.junit.Assert.*; + +import java.util.Arrays; + +import com.yahoo.vespa.defaults.Defaults; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.HostService; +import com.yahoo.vespa.serviceview.bindings.ModelResponse; +import com.yahoo.vespa.serviceview.bindings.ServicePort; +import com.yahoo.vespa.serviceview.bindings.ServiceView; + +/** + * Functional tests for the programmatic view of cloud.config.model. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ServiceModelTest { + + ServiceModel model; + + @Before + public void setUp() throws Exception { + ModelResponse model = syntheticModelResponse(); + this.model = new ServiceModel(model); + } + + static ModelResponse syntheticModelResponse() { + ModelResponse model = new ModelResponse(); + HostService h = new HostService(); + h.name = "vespa.yahoo.com"; + com.yahoo.vespa.serviceview.bindings.Service service0 = new com.yahoo.vespa.serviceview.bindings.Service(); + { + service0.clustername = "examplecluster"; + service0.clustertype = "somethingservers"; + service0.index = 1L; + service0.type = "something"; + service0.name = "examplename"; + service0.configid = "blblb/lbl.0"; + ServicePort port = new ServicePort(); + port.number = Defaults.getDefaults().vespaWebServicePort(); + port.tags = "state http"; + service0.ports = Arrays.asList(new ServicePort[] { port }); + } + com.yahoo.vespa.serviceview.bindings.Service service1 = new com.yahoo.vespa.serviceview.bindings.Service(); + { + service1.clustername = "examplecluster"; + service1.clustertype = "somethingservers"; + service1.index = 2L; + service1.type = "container-clustercontroller"; + service1.name = "clustercontroller"; + service1.configid = "clustercontroller/lbl.0"; + ServicePort port = new ServicePort(); + port.number = 4090; + port.tags = "state http"; + service1.ports = Arrays.asList(new ServicePort[] { port }); + } + com.yahoo.vespa.serviceview.bindings.Service service2 = new com.yahoo.vespa.serviceview.bindings.Service(); + { + service2.clustername = "tralala"; + service2.clustertype = "admin"; + service2.index = 3L; + service2.type = "configserver"; + service2.name = "configservername"; + service2.configid = "clustercontroller/lbl.0"; + ServicePort port = new ServicePort(); + port.number = 5000; + port.tags = "state http"; + service2.ports = Arrays.asList(new ServicePort[] { port }); + } + h.services = Arrays.asList(new com.yahoo.vespa.serviceview.bindings.Service[] { service0, service1, service2 }); + model.hosts = Arrays.asList(new HostService[] { h }); + return model; + } + + @After + public void tearDown() throws Exception { + model = null; + } + + @Test + public final void test() { + final String uriBase = "http://configserver:5000/"; + ApplicationView x = model.showAllClusters(uriBase, "/tenant/default/application/default"); + assertEquals(2, x.clusters.size()); + String urlTracking = null; + for (com.yahoo.vespa.serviceview.bindings.ClusterView c : x.clusters) { + for (ServiceView s : c.services) { + if ("examplename".equals(s.serviceName)) { + assertEquals("something", s.serviceType); + urlTracking = s.url; + break; + } + } + } + assertNotNull(urlTracking); + final String serviceIdentifier = urlTracking.substring(urlTracking.indexOf("something"), + urlTracking.length() - "/state/v1/".length()); + Service y = model.getService(serviceIdentifier); + assertEquals("examplename", y.name); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateResourceTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateResourceTest.java new file mode 100644 index 00000000000..0f577030984 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateResourceTest.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.serviceview; + +import static org.junit.Assert.*; + +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.client.Client; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.UriInfo; + +import com.yahoo.vespa.defaults.Defaults; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.container.jaxrs.annotation.Component; +import com.yahoo.vespa.serviceview.bindings.ApplicationView; +import com.yahoo.vespa.serviceview.bindings.HealthClient; +import com.yahoo.vespa.serviceview.bindings.ModelResponse; + +/** + * Functional test for {@link StateResource}. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class StateResourceTest { + + private static final String EXTERNAL_BASE_URI = "http://someserver:8080/serviceview/"; + + private static class TestResource extends StateResource { + private static final String BASE_URI = "http://vespa.yahoo.com:8080/state/v1"; + + TestResource(@Component ConfigServerLocation configServer, @Context UriInfo ui) { + super(configServer, ui); + } + + @Override + protected ModelResponse getModelConfig(String tenant, String application, String environment, String region, String instance) { + return ServiceModelTest.syntheticModelResponse(); + } + + @Override + protected HealthClient getHealthClient(String apiParams, Service s, int requestedPort, Client client) { + HealthClient healthClient = Mockito.mock(HealthClient.class); + HashMap<Object, Object> dummyHealthData = new HashMap<>(); + HashMap<String, String> dummyLink = new HashMap<>(); + dummyLink.put("url", BASE_URI); + dummyHealthData.put("resources", Arrays.asList(dummyLink)); + Mockito.when(healthClient.getHealthInfo()).thenReturn(dummyHealthData); + return healthClient; + } + } + + StateResource testResource; + ServiceModel correspondingModel; + + @Before + public void setUp() throws Exception { + UriInfo base = Mockito.mock(UriInfo.class); + Mockito.when(base.getBaseUri()).thenReturn(new URI(EXTERNAL_BASE_URI)); + ConfigServerLocation dummyLocation = new ConfigServerLocation(new ConfigserverConfig(new ConfigserverConfig.Builder())); + testResource = new TestResource(dummyLocation, base); + correspondingModel = new ServiceModel(ServiceModelTest.syntheticModelResponse()); + } + + @After + public void tearDown() throws Exception { + testResource = null; + correspondingModel = null; + } + + @SuppressWarnings("rawtypes") + @Test + public final void test() { + Service s = correspondingModel.resolve("vespa.yahoo.com", 8080, null); + String api = "/state/v1"; + HashMap boom = testResource.singleService("default", "default", "default", "default", "default", s.getIdentifier(8080), api); + assertEquals(EXTERNAL_BASE_URI + "v1/tenant/default/application/default/environment/default/region/default/instance/default/service/" + s.getIdentifier(8080) + api, + ((Map) ((List) boom.get("resources")).get(0)).get("url")); + } + + @Test + public final void testLinkEquality() { + ApplicationView explicitParameters = testResource.getUserInfo("default", "default", "default", "default", "default"); + ApplicationView implicitParameters = testResource.getDefaultUserInfo(); + assertEquals(explicitParameters.clusters.get(0).services.get(0).url, implicitParameters.clusters.get(0).services.get(0).url); + } + +} diff --git a/configserver/src/test/resources/configdefinitions/app.def b/configserver/src/test/resources/configdefinitions/app.def new file mode 100644 index 00000000000..cfc5041660c --- /dev/null +++ b/configserver/src/test/resources/configdefinitions/app.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=config + +message string default="Hello!" + +times int default=1 + +a[].name string diff --git a/configserver/src/test/resources/configdefinitions/datastructures.def b/configserver/src/test/resources/configdefinitions/datastructures.def new file mode 100644 index 00000000000..5ac8227bd13 --- /dev/null +++ b/configserver/src/test/resources/configdefinitions/datastructures.def @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=3 +namespace=config + +date[] string + +stock[].ticker string +stock[].type enum { COMMON, ETF, ETC } default=COMMON +stock[].volume[] int diff --git a/configserver/src/test/resources/configdefinitions/function-test.def b/configserver/src/test/resources/configdefinitions/function-test.def new file mode 100644 index 00000000000..4ddccdd4ddf --- /dev/null +++ b/configserver/src/test/resources/configdefinitions/function-test.def @@ -0,0 +1,54 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# +# This def file should test most aspects of def files that makes a difference +# for the autogenerated config classes. The goal is to trigger all blocks of +# code in the code generators. This includes: +# +# - Use all legal special characters in the def file name, to ensure that those +# that needs to be replaced in type names are actually replaced. +# - Use the same enum type twice to verify that we dont declare or define it +# twice. +# - Use the same struct type twice for the same reason. +# - Include arrays of primitives and structs. +# - Include enum primitives and array of enums. Arrays of enums must be handled +# specially by the C++ code. +# - Include enums both with and without default values. +# - Include primitive string, numbers & doubles both with and without default +# values. +# - Have an array within a struct, to verify that we correctly recurse. +# - Reuse type name further within to ensure that this works. + +version=4 +namespace=config + +# Some random bool without a default value. These comments exist to check + # that comment parsing works. +bool_val bool + ## A bool with a default value set. +bool_with_def bool default=false +int_val int +int_with_def int default=-545 +double_val double +double_with_def double default=-6.43 +# Another comment +string_val string +stringwithdef string default="foobar" +enum_val enum { FOO, BAR, FOOBAR } +enumwithdef enum { FOO2, BAR2, FOOBAR2 } default=BAR2 +refval reference +refwithdef reference default=":parent:" + +boolarr[] bool +intarr[] int +doublearr[] double +stringarr[] string +enumarr[] enum { ARRAY, VALUES } +refarr[] reference + + +myarray[].intval int default=14 +myarray[].stringval[] string +myarray[].enumval enum { INNER, ENUM, TYPE } default=TYPE +myarray[].refval reference # Value in array without default +myarray[].anotherarray[].foo int default=-4 + diff --git a/configserver/src/test/resources/configdefinitions/md5test.def b/configserver/src/test/resources/configdefinitions/md5test.def new file mode 100644 index 00000000000..100bc679b62 --- /dev/null +++ b/configserver/src/test/resources/configdefinitions/md5test.def @@ -0,0 +1,27 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# version=4 , version in comment does not count. + +# Added empty line to see if we can confuse +# the server's md5 calculation +version=3 +namespace=config + +#even adding a variable name starting with 'version' +versiontag int default=3 + +blabla string default="" +tabs string default=" " +test int + +# test multiple spaces/tabs +spaces int +singletab string +multitabs double + +# test enum +normal enum { VAL1, VAL2 } default=VAL1 +spacevalues enum { V1 , V2 , V3 , V4 } default=V3 + +# Comments and empty lines at the end + + diff --git a/configserver/src/test/resources/configdefinitions/simpletypes.def b/configserver/src/test/resources/configdefinitions/simpletypes.def new file mode 100644 index 00000000000..4609afa9f62 --- /dev/null +++ b/configserver/src/test/resources/configdefinitions/simpletypes.def @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Config containing only simple leaf types with default values, that can be used +# for testing individual types in detail. +version=1 +namespace=config + +boolval bool default=false +doubleval double default=0.0 +enumval enum { VAL1, VAL2 } default=VAL1 +intval int default=0 +longval long default=0 +stringval string default="s" diff --git a/configserver/src/test/resources/configdefinitions/unicode.def b/configserver/src/test/resources/configdefinitions/unicode.def new file mode 100644 index 00000000000..41100582edc --- /dev/null +++ b/configserver/src/test/resources/configdefinitions/unicode.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=2 +namespace=config + +unicodestring1 string +unicodestring2 string default="abc æøå 囲碁 ÆØÅ ABC" diff --git a/configserver/src/test/resources/deploy/advancedapp/external/foo/bar/lol b/configserver/src/test/resources/deploy/advancedapp/external/foo/bar/lol new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/configserver/src/test/resources/deploy/advancedapp/external/foo/bar/lol diff --git a/configserver/src/test/resources/deploy/advancedapp/hosts.xml b/configserver/src/test/resources/deploy/advancedapp/hosts.xml new file mode 100644 index 00000000000..3ab86a21aef --- /dev/null +++ b/configserver/src/test/resources/deploy/advancedapp/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> +</hosts> diff --git a/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd b/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd new file mode 100644 index 00000000000..29a22b8cf9f --- /dev/null +++ b/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd @@ -0,0 +1,13 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search keyvalue { + document keyvalue { + field value type string { + indexing: summary + } + } + document-summary keyvaluesummary { + summary value type string { + source: value + } + } +} diff --git a/configserver/src/test/resources/deploy/advancedapp/services.xml b/configserver/src/test/resources/deploy/advancedapp/services.xml new file mode 100644 index 00000000000..d2711b7f054 --- /dev/null +++ b/configserver/src/test/resources/deploy/advancedapp/services.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + <logserver hostalias="node1"/> + <slobroks> + <slobrok hostalias="node1"/> + </slobroks> + </admin> + + <jdisc version="1.0"> + <search /> + <nodes> + <node hostalias="node1" baseport='8000'/> + </nodes> + </jdisc> + + <content version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="keyvalue" mode="index"/> + </documents> + <nodes>> + <node hostalias="node1" distribution-key="0"/> + </nodes> + </content> + +</services> diff --git a/configserver/src/test/resources/deploy/app/services.xml b/configserver/src/test/resources/deploy/app/services.xml new file mode 100644 index 00000000000..f425d2f35d2 --- /dev/null +++ b/configserver/src/test/resources/deploy/app/services.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1" /> + </admin> + +</services> diff --git a/configserver/src/test/resources/deploy/validapp/hosts.xml b/configserver/src/test/resources/deploy/validapp/hosts.xml new file mode 100644 index 00000000000..3ab86a21aef --- /dev/null +++ b/configserver/src/test/resources/deploy/validapp/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="localhost"> + <alias>node1</alias> + </host> +</hosts> diff --git a/configserver/src/test/resources/deploy/validapp/services.xml b/configserver/src/test/resources/deploy/validapp/services.xml new file mode 100644 index 00000000000..f425d2f35d2 --- /dev/null +++ b/configserver/src/test/resources/deploy/validapp/services.xml @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1" /> + </admin> + +</services> |