diff options
Diffstat (limited to 'configserver')
23 files changed, 547 insertions, 519 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 6ed1ddf6c7e..a5852d1dd8a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -2,16 +2,27 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; + import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.Deployment; +import com.yahoo.vespa.config.server.http.ContentHandler; +import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.http.v2.ApplicationContentRequest; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ActivateLock; import com.yahoo.vespa.config.server.tenant.Rotations; @@ -19,6 +30,8 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.curator.Curator; +import java.io.IOException; +import java.net.URI; import java.time.Clock; import java.time.Duration; import java.util.Optional; @@ -34,20 +47,29 @@ import java.util.logging.Logger; public class ApplicationRepository implements com.yahoo.config.provision.Deployer { private static final Logger log = Logger.getLogger(ApplicationRepository.class.getName()); - + private final Tenants tenants; private final Optional<Provisioner> hostProvisioner; private final ConfigserverConfig configserverConfig; private final Curator curator; + private final LogServerLogGrabber logServerLogGrabber; + private final ApplicationConvergenceChecker convergeChecker; + private final ContentHandler contentHandler = new ContentHandler(); private final Clock clock; private final DeployLogger logger = new SilentDeployLogger(); - public ApplicationRepository(Tenants tenants, HostProvisionerProvider hostProvisionerProvider, - ConfigserverConfig configserverConfig, Curator curator) { + public ApplicationRepository(Tenants tenants, + HostProvisionerProvider hostProvisionerProvider, + ConfigserverConfig configserverConfig, + Curator curator, + LogServerLogGrabber logServerLogGrabber, + ApplicationConvergenceChecker applicationConvergenceChecker) { this.tenants = tenants; this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); this.configserverConfig = configserverConfig; this.curator = curator; + this.logServerLogGrabber = logServerLogGrabber; + this.convergeChecker = applicationConvergenceChecker; this.clock = Clock.systemUTC(); } @@ -74,7 +96,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye configserverConfig, hostProvisioner, new ActivateLock(curator, tenant.getPath()), - timeout, + timeout, clock, /* already deployed, validate: */ false)); } @@ -87,9 +109,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye timeout, clock); } - /** + /** * Removes a previously deployed application - * + * * @return true if the application was found and removed, false if it was not present * @throws RuntimeException if the remove transaction fails. This method is exception safe. */ @@ -99,7 +121,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye TenantApplications tenantApplications = owner.get().getApplicationRepo(); if ( ! tenantApplications.listApplications().contains(applicationId)) return false; - + // TODO: Push lookup logic down long sessionId = tenantApplications.getSessionIdForApplication(applicationId); LocalSessionRepo localSessionRepo = owner.get().getLocalSessionRepo(); @@ -122,4 +144,40 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return true; } + public String grabLog(Tenant tenant, ApplicationId applicationId) { + Application application = getApplication(tenant, applicationId); + return logServerLogGrabber.grabLog(application); + } + + public HttpResponse nodeConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) { + Application application = getApplication(tenant, applicationId); + return convergeChecker.nodeConvergenceCheck(application, hostname, uri); + } + + public void waitForConfigConverged(Tenant tenant, ApplicationId applicationId, TimeoutBudget timeoutBudget) throws IOException { + Application application = getApplication(tenant, applicationId); + convergeChecker.waitForConfigConverged(application, timeoutBudget); + } + + public HttpResponse listConfigConvergence(Tenant tenant, ApplicationId applicationId, URI uri) { + Application application = getApplication(tenant, applicationId); + return convergeChecker.listConfigConvergence(application, uri); + } + + public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) { + return getApplication(tenant, applicationId).getApplicationGeneration(); + } + + public HttpResponse getContent(Tenant tenant, ApplicationId applicationId, Zone zone, HttpRequest request) { + LocalSession session = SessionHandler.getSessionFromRequest(tenant.getLocalSessionRepo(), + tenant.getApplicationRepo().getSessionIdForApplication(applicationId)); + return contentHandler.get(ApplicationContentRequest.create(request, session, applicationId, zone)); + } + + private Application getApplication(Tenant tenant, ApplicationId applicationId) { + long sessionId = tenant.getApplicationRepo().getSessionIdForApplication(applicationId); + RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0); + return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty()); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 88489aaa17b..214f0defedf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -1,167 +1,86 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.google.inject.Inject; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.provision.Version; -import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.GenerationCounter; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.server.model.SuperModel; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactoryFactory; import java.io.IOException; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.io.StringReader; /** - * Controls the lifetime of the {@link SuperModel} and the {@link SuperModelRequestHandler}. + * Handler for global configs that must be resolved using the global SuperModel instance. Deals with + * reloading of config as well. * * @author lulf * @since 5.9 */ -public class SuperModelController implements RequestHandler { +public class SuperModelController { - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SuperModelController.class.getName()); - private volatile SuperModelRequestHandler handler; - private final GenerationCounter generationCounter; - private final Zone zone; - private final long masterGeneration; + private final SuperModel model; + private final long generation; private final ConfigDefinitionRepo configDefinitionRepo; private final ConfigResponseFactory responseFactory; - private volatile boolean enabled = false; - /** - * Creates a supermodel controller - * - * @param generationCounter this will be the SuperModelGenerationCounter in production - */ - @Inject - public SuperModelController(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, ConfigserverConfig configserverConfig) { - this.generationCounter = generationCounter; + public SuperModelController(SuperModel model, ConfigDefinitionRepo configDefinitionRepo, long generation, ConfigResponseFactory responseFactory) { + this.model = model; this.configDefinitionRepo = configDefinitionRepo; - this.masterGeneration = configserverConfig.masterGeneration(); - this.responseFactory = ConfigResponseFactoryFactory.createFactory(configserverConfig); - this.zone = new Zone(configserverConfig); - this.handler = createNewHandler(Collections.emptyMap()); + this.generation = generation; + this.responseFactory = responseFactory; } /** - * Signals that config has been reloaded for an {@link com.yahoo.vespa.config.server.application.Application} - * belonging to a tenant. + * Resolves global config for given request. * - * TODO: This is a bit too complex I think. - * - * @param tenant Name of tenant owning the application. - * @param applicationSet The reloaded set of {@link com.yahoo.vespa.config.server.application.Application}. + * @param request The {@link com.yahoo.vespa.config.GetConfigRequest} to find config for. + * @return a {@link com.yahoo.vespa.config.protocol.ConfigResponse} containing the response for this request. + * @throws java.lang.IllegalArgumentException if no such config was found. */ - public synchronized void reloadConfig(TenantName tenant, ApplicationSet applicationSet) { - Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); - if (!newModels.containsKey(tenant)) { - newModels.put(tenant, new LinkedHashMap<>()); + public ConfigResponse resolveConfig(GetConfigRequest request) { + ConfigKey<?> configKey = request.getConfigKey(); + InnerCNode targetDef = getConfigDefinition(request.getConfigKey(), request.getDefContent()); + try { + ConfigPayload payload = model.getConfig(configKey); + return responseFactory.createResponse(payload, targetDef, generation); + } catch (IOException e) { + throw new ConfigurationRuntimeException("Unable to resolve config", e); } - // TODO: Should supermodel care about multiple versions? - newModels.get(tenant).put(applicationSet.getId(), applicationSet.getForVersionOrLatest(Optional.empty())); - handler = createNewHandler(newModels); } - public synchronized void removeApplication(ApplicationId applicationId) { - Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); - if (newModels.containsKey(applicationId.tenant())) { - newModels.get(applicationId.tenant()).remove(applicationId); - if (newModels.get(applicationId.tenant()).isEmpty()) { - newModels.remove(applicationId.tenant()); + private InnerCNode getConfigDefinition(ConfigKey<?> configKey, DefContent defContent) { + if (defContent.isEmpty()) { + ConfigDefinitionKey configDefinitionKey = new ConfigDefinitionKey(configKey.getName(), configKey.getNamespace()); + ConfigDefinition configDefinition = configDefinitionRepo.getConfigDefinitions().get(configDefinitionKey); + if (configDefinition == null) { + throw new UnknownConfigDefinitionException("Unable to find config definition for '" + configKey.getNamespace() + "." + configKey.getName()); } - } - handler = createNewHandler(newModels); - } - - private SuperModelRequestHandler createNewHandler(Map<TenantName, Map<ApplicationId, Application>> newModels) { - long generation = generationCounter.get() + masterGeneration; - SuperModel model = new SuperModel(newModels, zone); - return new SuperModelRequestHandler(model, configDefinitionRepo, generation, responseFactory); - } - - private Map<TenantName, Map<ApplicationId, Application>> getCurrentModels() { - if (handler != null) { - return handler.getSuperModel().getCurrentModels(); + return configDefinition.getCNode(); } else { - return new LinkedHashMap<>(); + DefParser dParser = new DefParser(configKey.getName(), new StringReader(defContent.asString())); + return dParser.getTree(); } } - private Map<TenantName, Map<ApplicationId, Application>> createModelCopy() { - Map<TenantName, Map<ApplicationId, Application>> currentModels = getCurrentModels(); - Map<TenantName, Map<ApplicationId, Application>> newModels = new LinkedHashMap<>(); - for (Map.Entry<TenantName, Map<ApplicationId, Application>> entry : currentModels.entrySet()) { - Map<ApplicationId, Application> appMap = new LinkedHashMap<>(); - newModels.put(entry.getKey(), appMap); - for (Map.Entry<ApplicationId, Application> appEntry : entry.getValue().entrySet()) { - appMap.put(appEntry.getKey(), appEntry.getValue()); - } - } - return newModels; + SuperModel getSuperModel() { + return model; } - public SuperModelRequestHandler getHandler() { return handler; } - - @Override - public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { - log.log(LogLevel.DEBUG, "SuperModelController resolving " + req + " for app id '" + appId + "'"); - if (handler != null) { - return handler.resolveConfig(req); - } - return null; - } + long getGeneration() { return generation; } public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { - return handler.getConfig(configClass, applicationId, configId); - } - - @Override - public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> key, boolean recursive) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { - throw new UnsupportedOperationException(); + return model.getConfig(configClass, applicationId, configId); } - @Override - public Set<String> allConfigIds(ApplicationId appID, Optional<Version> vespaVersion) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { - return enabled && appId.equals(ApplicationId.global()); - } - - @Override - public ApplicationId resolveApplicationId(String hostName) { - return ApplicationId.global(); - } - - public void enable() { - enabled = true; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index b08ac0827a1..9291e6030e2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -1,84 +1,166 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.ConfigurationRuntimeException; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.model.SuperModel; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; +import com.yahoo.vespa.config.server.rpc.ConfigResponseFactoryFactory; import java.io.IOException; -import java.io.StringReader; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; /** - * Handler for global configs that must be resolved using the global SuperModel instance. Deals with - * reloading of config as well. + * Handles request for supermodel config * * @author lulf * @since 5.9 */ -public class SuperModelRequestHandler { - private final SuperModel model; - private final long generation; +public class SuperModelRequestHandler implements RequestHandler { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SuperModelRequestHandler.class.getName()); + private volatile SuperModelController handler; + private final GenerationCounter generationCounter; + private final Zone zone; + private final long masterGeneration; private final ConfigDefinitionRepo configDefinitionRepo; private final ConfigResponseFactory responseFactory; + private volatile boolean enabled = false; - public SuperModelRequestHandler(SuperModel model, ConfigDefinitionRepo configDefinitionRepo, long generation, ConfigResponseFactory responseFactory) { - this.model = model; + /** + * Creates a supermodel controller + */ + @Inject + public SuperModelRequestHandler(GenerationCounter generationCounter, ConfigDefinitionRepo configDefinitionRepo, + ConfigserverConfig configserverConfig) { + this.generationCounter = generationCounter; this.configDefinitionRepo = configDefinitionRepo; - this.generation = generation; - this.responseFactory = responseFactory; + this.masterGeneration = configserverConfig.masterGeneration(); + this.responseFactory = ConfigResponseFactoryFactory.createFactory(configserverConfig); + this.zone = new Zone(configserverConfig); + this.handler = createNewHandler(Collections.emptyMap()); } /** - * Resolves global config for given request. + * Signals that config has been reloaded for an {@link com.yahoo.vespa.config.server.application.Application} + * belonging to a tenant. + * + * TODO: This is a bit too complex I think. * - * @param request The {@link com.yahoo.vespa.config.GetConfigRequest} to find config for. - * @return a {@link com.yahoo.vespa.config.protocol.ConfigResponse} containing the response for this request. - * @throws java.lang.IllegalArgumentException if no such config was found. + * @param tenant Name of tenant owning the application. + * @param applicationSet The reloaded set of {@link com.yahoo.vespa.config.server.application.Application}. */ - public ConfigResponse resolveConfig(GetConfigRequest request) { - ConfigKey<?> configKey = request.getConfigKey(); - InnerCNode targetDef = getConfigDefinition(request.getConfigKey(), request.getDefContent()); - try { - ConfigPayload payload = model.getConfig(configKey); - return responseFactory.createResponse(payload, targetDef, generation); - } catch (IOException e) { - throw new ConfigurationRuntimeException("Unable to resolve config", e); + public synchronized void reloadConfig(TenantName tenant, ApplicationSet applicationSet) { + Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); + if (!newModels.containsKey(tenant)) { + newModels.put(tenant, new LinkedHashMap<>()); } + // TODO: Should supermodel care about multiple versions? + newModels.get(tenant).put(applicationSet.getId(), applicationSet.getForVersionOrLatest(Optional.empty())); + handler = createNewHandler(newModels); } - private InnerCNode getConfigDefinition(ConfigKey<?> configKey, DefContent defContent) { - if (defContent.isEmpty()) { - ConfigDefinitionKey configDefinitionKey = new ConfigDefinitionKey(configKey.getName(), configKey.getNamespace()); - ConfigDefinition configDefinition = configDefinitionRepo.getConfigDefinitions().get(configDefinitionKey); - if (configDefinition == null) { - throw new UnknownConfigDefinitionException("Unable to find config definition for '" + configKey.getNamespace() + "." + configKey.getName()); + public synchronized void removeApplication(ApplicationId applicationId) { + Map<TenantName, Map<ApplicationId, Application>> newModels = createModelCopy(); + if (newModels.containsKey(applicationId.tenant())) { + newModels.get(applicationId.tenant()).remove(applicationId); + if (newModels.get(applicationId.tenant()).isEmpty()) { + newModels.remove(applicationId.tenant()); } - return configDefinition.getCNode(); + } + handler = createNewHandler(newModels); + } + + private SuperModelController createNewHandler(Map<TenantName, Map<ApplicationId, Application>> newModels) { + long generation = generationCounter.get() + masterGeneration; + SuperModel model = new SuperModel(newModels, zone); + return new SuperModelController(model, configDefinitionRepo, generation, responseFactory); + } + + private Map<TenantName, Map<ApplicationId, Application>> getCurrentModels() { + if (handler != null) { + return handler.getSuperModel().applicationModels(); } else { - DefParser dParser = new DefParser(configKey.getName(), new StringReader(defContent.asString())); - return dParser.getTree(); + return new LinkedHashMap<>(); } } - SuperModel getSuperModel() { - return model; + private Map<TenantName, Map<ApplicationId, Application>> createModelCopy() { + Map<TenantName, Map<ApplicationId, Application>> currentModels = getCurrentModels(); + Map<TenantName, Map<ApplicationId, Application>> newModels = new LinkedHashMap<>(); + for (Map.Entry<TenantName, Map<ApplicationId, Application>> entry : currentModels.entrySet()) { + Map<ApplicationId, Application> appMap = new LinkedHashMap<>(); + newModels.put(entry.getKey(), appMap); + for (Map.Entry<ApplicationId, Application> appEntry : entry.getValue().entrySet()) { + appMap.put(appEntry.getKey(), appEntry.getValue()); + } + } + return newModels; } - long getGeneration() { return generation; } + public SuperModelController getHandler() { return handler; } + + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + log.log(LogLevel.DEBUG, "SuperModelRequestHandler resolving " + req + " for app id '" + appId + "'"); + if (handler != null) { + return handler.resolveConfig(req); + } + return null; + } public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { - return model.getConfig(configClass, applicationId, configId); + return handler.getConfig(configClass, applicationId, configId); + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + throw new UnsupportedOperationException(); + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> key, boolean recursive) { + throw new UnsupportedOperationException(); + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + throw new UnsupportedOperationException(); + } + + @Override + public Set<String> allConfigIds(ApplicationId appID, Optional<Version> vespaVersion) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + return enabled && appId.equals(ApplicationId.global()); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + return ApplicationId.global(); + } + + public void enable() { + enabled = true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java index 2ce95e016f4..fdec3939f2f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/LogServerLogGrabber.java @@ -4,19 +4,14 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ModelConfig; import com.yahoo.component.AbstractComponent; import com.google.inject.Inject; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.yolean.Exceptions; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.PrintWriter; import java.net.Socket; -import java.nio.charset.StandardCharsets; import java.util.Optional; /** @@ -28,7 +23,6 @@ import java.util.Optional; public class LogServerLogGrabber extends AbstractComponent { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(LogServerLogGrabber.class.getName()); - @Inject public LogServerLogGrabber() {} private Optional<Integer> getErrorLogPort(ModelConfig.Hosts.Services service) { @@ -43,7 +37,7 @@ public class LogServerLogGrabber extends AbstractComponent { int port; } - public HttpResponse grabLog(Application application) { + public String grabLog(Application application) { final ModelConfig config; try { @@ -61,7 +55,7 @@ public class LogServerLogGrabber extends AbstractComponent { Optional<Integer> logPort = getErrorLogPort(logService); if (logPort.isPresent()) { if (logServerConnectionInfo.hostName != null) { - throw new RuntimeException("Found several log server ports."); + throw new RuntimeException("Found several log server ports"); } logServerConnectionInfo.hostName = host.name(); logServerConnectionInfo.port = logPort.get(); @@ -69,14 +63,7 @@ public class LogServerLogGrabber extends AbstractComponent { })); if (logServerConnectionInfo.hostName == null) { - return new HttpResponse(503) { - @Override - public void render(OutputStream outputStream) throws IOException { - PrintWriter printWriter = new PrintWriter(outputStream); - printWriter.print("Did not find any log server in config model."); - printWriter.close(); - } - }; + throw new InternalServerException("Did not find any log server in config model"); } log.log(LogLevel.DEBUG, "Requested error logs, pulling from logserver on " + logServerConnectionInfo.hostName + " " + logServerConnectionInfo.port); @@ -85,20 +72,9 @@ public class LogServerLogGrabber extends AbstractComponent { response = readLog(logServerConnectionInfo.hostName, logServerConnectionInfo.port); log.log(LogLevel.DEBUG, "Requested error logs was " + response.length() + " characters"); } catch (IOException e) { - return HttpErrorResponse.internalServerError(Exceptions.toMessageString(e)); + throw new InternalServerException(Exceptions.toMessageString(e)); } - - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write(response.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } - }; + return response; } private String readLog(String host, int port) throws IOException { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 58d651ae33a..37cea22e420 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -43,12 +43,4 @@ public class FileDBRegistry implements FileRegistry { return entries; } - @Override - public Set<String> allRelativePaths() { - Set<String> ret = new HashSet<>(); - for (Entry entry : entries) { - ret.add(entry.relativePath); - } - return ret; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java index cc4689682dd..b54c722a7e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionActiveHandlerBase.java @@ -1,39 +1,38 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.yahoo.config.provision.Provisioner; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.logging.AccessLog; import com.yahoo.vespa.config.server.tenant.ActivateLock; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.deploy.Deployment; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import java.util.Optional; import java.util.concurrent.Executor; /** * @author lulf */ public class SessionActiveHandlerBase extends SessionHandler { + private final ApplicationRepository applicationRepository; - public SessionActiveHandlerBase(Executor executor, AccessLog accessLog) { + public SessionActiveHandlerBase(Executor executor, AccessLog accessLog, ApplicationRepository applicationRepository ) { super(executor, accessLog); + this.applicationRepository = applicationRepository; } protected void activate(HttpRequest request, LocalSessionRepo localSessionRepo, ActivateLock activateLock, TimeoutBudget timeoutBudget, - Optional<Provisioner> hostProvisioner, LocalSession localSession) { - // TODO: Use an injected applicationRepository from the callers of this instead - // TODO: And then get rid of the activateLock and localSessionRepo arguments in deployFromPreparedSession - ApplicationRepository applicationRepository = new ApplicationRepository(null, HostProvisionerProvider.from(hostProvisioner), null, null); - Deployment deployment = applicationRepository.deployFromPreparedSession(localSession, activateLock, localSessionRepo, timeoutBudget.timeLeft()); + // TODO: Get rid of the activateLock and localSessionRepo arguments in deployFromPreparedSession + Deployment deployment = applicationRepository.deployFromPreparedSession(localSession, + activateLock, + localSessionRepo, + timeoutBudget.timeLeft()); deployment.setIgnoreLockFailure(shouldIgnoreLockFailure(request)); deployment.setIgnoreSessionStaleFailure(shouldIgnoreSessionStaleFailure(request)); deployment.activate(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java index ba7eff7c461..2b5bc4b3d35 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentRequest.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.config.server.session.LocalSession; * @author lulf * @since 5.3 */ -class ApplicationContentRequest extends ContentRequest { +public class ApplicationContentRequest extends ContentRequest { private static final String uriPattern = "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*"; private final ApplicationId applicationId; @@ -27,7 +27,7 @@ class ApplicationContentRequest extends ContentRequest { this.zone = zone; } - static ContentRequest create(HttpRequest request, LocalSession session, ApplicationId applicationId, Zone zone) { + public static ContentRequest create(HttpRequest request, LocalSession session, ApplicationId applicationId, Zone zone) { return new ApplicationContentRequest(request, session, applicationId, zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index c0482ad5d99..e9432e9cf81 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.HostFilter; @@ -13,28 +12,22 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; +import com.yahoo.vespa.config.server.http.HttpConfigResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; -import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; -import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; -import com.yahoo.vespa.curator.Curator; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.time.Clock; import java.time.Duration; import java.util.List; @@ -42,7 +35,7 @@ import java.util.Optional; import java.util.concurrent.Executor; /** - * Handler for deleting a currently active application for a tenant. + * Operations on applications (delete, wait for config convergence, restart, application content etc.) * * @author hmusum * @since 5.4 @@ -52,25 +45,21 @@ public class ApplicationHandler extends HttpHandler { private static final String REQUEST_PROPERTY_TIMEOUT = "timeout"; private final Tenants tenants; - private final ContentHandler contentHandler = new ContentHandler(); + private final Optional<Provisioner> hostProvisioner; - private final ApplicationConvergenceChecker convergeChecker; private final Zone zone; - private final LogServerLogGrabber logServerLogGrabber; private final ApplicationRepository applicationRepository; - public ApplicationHandler(Executor executor, AccessLog accessLog, Tenants tenants, - HostProvisionerProvider hostProvisionerProvider, Zone zone, - ApplicationConvergenceChecker convergeChecker, - LogServerLogGrabber logServerLogGrabber, - ConfigserverConfig configserverConfig, Curator curator) { + public ApplicationHandler(Executor executor, AccessLog accessLog, + Tenants tenants, + HostProvisionerProvider hostProvisionerProvider, + Zone zone, + ApplicationRepository applicationRepository) { super(executor, accessLog); this.tenants = tenants; this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); this.zone = zone; - this.convergeChecker = convergeChecker; - this.logServerLogGrabber = logServerLogGrabber; - this.applicationRepository = new ApplicationRepository(tenants, hostProvisionerProvider, configserverConfig, curator); + this.applicationRepository = applicationRepository; } @Override @@ -89,27 +78,24 @@ public class ApplicationHandler extends HttpHandler { Tenant tenant = verifyTenantAndApplication(applicationId); if (isServiceConvergeRequest(request)) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.nodeConvergenceCheck(application, getHostFromRequest(request), request.getUri()); + return applicationRepository.nodeConvergenceCheck(tenant, applicationId, getHostFromRequest(request), request.getUri()); } if (isContentRequest(request)) { - LocalSession session = SessionHandler.getSessionFromRequest(tenant.getLocalSessionRepo(), tenant.getApplicationRepo().getSessionIdForApplication(applicationId)); - return contentHandler.get(ApplicationContentRequest.create(request, session, applicationId, zone)); + return applicationRepository.getContent(tenant, applicationId, zone, request); } - Application application = getApplication(tenant, applicationId); // TODO: Remove this once the config convergence logic is moved to client and is live for all clusters. if (isConvergeRequest(request)) { try { - convergeChecker.waitForConfigConverged(application, new TimeoutBudget(Clock.systemUTC(), durationFromRequestTimeout(request))); + applicationRepository.waitForConfigConverged(tenant, applicationId, new TimeoutBudget(Clock.systemUTC(), durationFromRequestTimeout(request))); } catch (IOException e) { throw new RuntimeException(e); } } if (isServiceConvergeListRequest(request)) { - return convergeChecker.listConfigConvergence(application, request.getUri()); + return applicationRepository.listConfigConvergence(tenant, applicationId, request.getUri()); } - return new GetApplicationResponse(Response.Status.OK, application.getApplicationGeneration()); + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId)); } @Override @@ -136,8 +122,18 @@ public class ApplicationHandler extends HttpHandler { if (getBindingMatch(request).groupCount() != 7) throw new NotFoundException("Illegal POST log request '" + request.getUri() + "': Must have 6 arguments but had " + ( getBindingMatch(request).groupCount()-1 ) ); - Application application = getApplication(tenant, applicationId); - return logServerLogGrabber.grabLog(application); + final String response = applicationRepository.grabLog(tenant, applicationId); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write(response.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String getContentType() { + return HttpConfigResponse.JSON_CONTENT_TYPE; + } + }; } private HostFilter hostFilterFrom(HttpRequest request) { @@ -164,13 +160,6 @@ public class ApplicationHandler extends HttpHandler { return Duration.ofSeconds(timeoutInSeconds); } - private Application getApplication(Tenant tenant, ApplicationId applicationId) { - TenantApplications applicationRepo = tenant.getApplicationRepo(); - RemoteSessionRepo remoteSessionRepo = tenant.getRemoteSessionRepo(); - long sessionId = applicationRepo.getSessionIdForApplication(applicationId); - RemoteSession session = remoteSessionRepo.getSession(sessionId, 0); - return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty()); - } private List<ApplicationId> listApplicationIds(Tenant tenant) { TenantApplications applicationRepo = tenant.getApplicationRepo(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 89463a0b8ee..af8288374f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -1,24 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import java.util.Optional; import java.util.concurrent.Executor; import com.google.inject.Inject; -import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.Tenants; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionActiveHandlerBase; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; -import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; /** @@ -30,18 +28,16 @@ import com.yahoo.vespa.config.server.session.LocalSession; public class SessionActiveHandler extends SessionActiveHandlerBase { private final Tenants tenants; - private final Optional<Provisioner> hostProvisioner; private final Zone zone; @Inject public SessionActiveHandler(Executor executor, AccessLog accessLog, Tenants tenants, - HostProvisionerProvider hostProvisionerProvider, - Zone zone) { - super(executor, accessLog); + Zone zone, + ApplicationRepository applicationRepository) { + super(executor, accessLog, applicationRepository); this.tenants = tenants; - this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); this.zone = zone; } @@ -52,7 +48,7 @@ public class SessionActiveHandler extends SessionActiveHandlerBase { log.log(LogLevel.DEBUG, "Found tenant '" + tenantName + "' in request"); Tenant tenant = Utils.checkThatTenantExists(tenants, tenantName); LocalSession localSession = getSessionFromRequestV2(tenant.getLocalSessionRepo(), request); - activate(request, tenant.getLocalSessionRepo(), tenant.getActivateLock(), timeoutBudget, hostProvisioner, localSession); + activate(request, tenant.getLocalSessionRepo(), tenant.getActivateLock(), timeoutBudget, localSession); return new SessionActiveResponse(localSession.getMetaData().getSlime(), tenantName, request, localSession, zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java index c6aa5ed7f8e..e22f4a42776 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModel.java @@ -17,7 +17,7 @@ import java.util.Collections; import java.util.Map; /** - * A config model that spans across all applications of all tenants in the config server. + * A config model that provides config containing information from all known tenants and applications. * * @author vegardh * @since 5.9 @@ -28,8 +28,8 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod private final LbServicesProducer lbProd; private final RoutingProducer zoneProd; - public SuperModel(Map<TenantName, Map<ApplicationId, Application>> newModels, Zone zone) { - this.models = newModels; + public SuperModel(Map<TenantName, Map<ApplicationId, Application>> models, Zone zone) { + this.models = models; this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(models), zone); this.zoneProd = new RoutingProducer(Collections.unmodifiableMap(models)); } @@ -49,9 +49,7 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod } } - public Map<TenantName, Map<ApplicationId, Application>> getCurrentModels() { - return models; - } + public Map<TenantName, Map<ApplicationId, Application>> applicationModels() { return models; } @Override public void getConfig(LbServicesConfig.Builder builder) { @@ -63,7 +61,8 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod zoneProd.getConfig(builder); } - public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, + ApplicationId applicationId, String configId) throws IOException { TenantName tenant = applicationId.tenant(); if (!models.containsKey(tenant)) { throw new IllegalArgumentException("Tenant " + tenant + " not found"); @@ -77,4 +76,5 @@ public class SuperModel implements LbServicesConfig.Producer, RoutingConfig.Prod ConfigPayload payload = application.getModel().getConfig(key, (ConfigDefinition)null, null); return payload.toInstance(configClass, configId); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 99036ee0027..1b32d6bde22 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -120,13 +120,12 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { ConfigserverConfig configserverConfig, Zone zone, Set<Rotation> rotations) { - return new ModelContextImpl.Properties( - applicationId, - configserverConfig.multitenant(), - ConfigServerSpec.fromConfig(configserverConfig), - configserverConfig.hostedVespa(), - zone, - rotations); + return new ModelContextImpl.Properties(applicationId, + configserverConfig.multitenant(), + ConfigServerSpec.fromConfig(configserverConfig), + configserverConfig.hostedVespa(), + zone, + rotations); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index cacd53cf945..9c1b2b4681e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -91,11 +91,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P this.applicationId = params.getApplicationId(); this.rotations = new Rotations(curator, tenantPath); this.rotationsSet = getRotations(params.rotations()); - this.properties = createModelContextProperties( - params.getApplicationId(), - configserverConfig, - zone, - rotationsSet); + this.properties = createModelContextProperties(params.getApplicationId(), configserverConfig, zone, rotationsSet); } /** Construct with all dependencies passed separately */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 0b08279f3ab..7afa9b7db87 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -24,13 +24,13 @@ import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.vespa.config.server.SuperModelRequestHandler; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.GetConfigContext; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.ReloadListener; import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.SuperModelController; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory; import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; @@ -79,7 +79,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private final HostRegistry<TenantName> hostRegistry; private final Map<TenantName, TenantHandlerProvider> tenantProviders = new ConcurrentHashMap<>(); - private final SuperModelController superModelController; + private final SuperModelRequestHandler superModelRequestHandler; private final MetricUpdater metrics; private final MetricUpdaterFactory metricUpdaterFactory; private final HostLivenessTracker hostLivenessTracker; @@ -93,9 +93,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { * @param config The config to use for setting up this server */ @Inject - public RpcServer(ConfigserverConfig config, SuperModelController superModelController, MetricUpdaterFactory metrics, + public RpcServer(ConfigserverConfig config, SuperModelRequestHandler superModelRequestHandler, MetricUpdaterFactory metrics, HostRegistries hostRegistries, HostLivenessTracker hostLivenessTracker) { - this.superModelController = superModelController; + this.superModelRequestHandler = superModelRequestHandler; this.metricUpdaterFactory = metrics; this.supervisor.setMaxOutputBufferSize(config.maxoutputbuffersize()); this.metrics = metrics.getOrCreateMetricUpdater(Collections.<String, String>emptyMap()); @@ -188,13 +188,13 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { */ @Override public void configReloaded(TenantName tenant, ApplicationSet applicationSet) { - final ApplicationId applicationId = applicationSet.getId(); + ApplicationId applicationId = applicationSet.getId(); configReloaded(delayedConfigResponses.drainQueue(applicationId), Tenants.logPre(applicationId)); reloadSuperModel(tenant, applicationSet); } private void reloadSuperModel(TenantName tenant, ApplicationSet applicationSet) { - superModelController.reloadConfig(tenant, applicationSet); + superModelRequestHandler.reloadConfig(tenant, applicationSet); configReloaded(delayedConfigResponses.drainQueue(ApplicationId.global()), Tenants.logPre(ApplicationId.global())); } @@ -253,7 +253,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void applicationRemoved(ApplicationId applicationId) { - superModelController.removeApplication(applicationId); + superModelRequestHandler.removeApplication(applicationId); configReloaded(delayedConfigResponses.drainQueue(applicationId), Tenants.logPre(applicationId)); configReloaded(delayedConfigResponses.drainQueue(ApplicationId.global()), Tenants.logPre(ApplicationId.global())); } @@ -286,12 +286,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } public ConfigResponse resolveConfig(JRTServerConfigRequest request, GetConfigContext context, Optional<Version> vespaVersion) { - Trace trace = context.trace(); - if (trace.shouldTrace(TRACELEVEL)) { - trace.trace(TRACELEVEL, "RpcServer.resolveConfig()"); - } - RequestHandler handler = context.requestHandler(); - return handler.resolveConfig(context.applicationId(), request, vespaVersion); + context.trace().trace(TRACELEVEL, "RpcServer.resolveConfig()"); + return context.requestHandler().resolveConfig(context.applicationId(), request, vespaVersion); } protected Supervisor getSupervisor() { @@ -338,7 +334,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { */ public GetConfigContext createGetConfigContext(Optional<TenantName> optionalTenant, JRTServerConfigRequest request, Trace trace) { if ("*".equals(request.getConfigKey().getConfigId())) { - return GetConfigContext.create(ApplicationId.global(), superModelController, trace); + return GetConfigContext.create(ApplicationId.global(), superModelRequestHandler, trace); } TenantName tenant = optionalTenant.orElse(TenantName.defaultName()); // perhaps needed for non-hosted? if ( ! hasRequestHandler(tenant)) { @@ -383,7 +379,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void onTenantsLoaded() { allTenantsLoaded = true; - superModelController.enable(); + superModelRequestHandler.enable(); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index b10865f257b..d2ded8ee226 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -163,10 +163,7 @@ public class SessionPreparer { void preprocess() { try { - this.applicationPackage = context.getApplicationPackage().preprocess( - properties.zone(), - null, - logger); + this.applicationPackage = context.getApplicationPackage().preprocess(properties.zone(), null, logger); } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) { throw new RuntimeException("Error deploying application package", e); } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 28652e73007..8a4069a4341 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -12,7 +12,7 @@ <component id="com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.SuperModelGenerationCounter" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.session.SessionPreparer" bundle="configserver" /> - <component id="com.yahoo.vespa.config.server.SuperModelController" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.SuperModelRequestHandler" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.StaticConfigDefinitionRepo" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.provision.HostProvisionerProvider" bundle="configserver" /> <component id="com.yahoo.vespa.curator.Curator" bundle="configserver" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index 9077b3fdac9..68b9c06aa7b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -1,29 +1,37 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.*; +import com.yahoo.jrt.Request; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; +import com.yahoo.vespa.config.protocol.CompressionType; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.model.SuperModel; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; import com.yahoo.vespa.model.VespaModel; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Optional; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author lulf @@ -31,100 +39,93 @@ import static org.junit.Assert.*; */ public class SuperModelControllerTest { - private static final File testApp = new File("src/test/resources/deploy/app"); - private SuperModelGenerationCounter counter; - private SuperModelController controller; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + private SuperModelController handler; @Before - public void setup() throws IOException { - counter = new SuperModelGenerationCounter(new MockCurator()); - controller = new SuperModelController(counter, - new TestConfigDefinitionRepo(), - new ConfigserverConfig(new ConfigserverConfig.Builder())); + public void setupHandler() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); + models.put(TenantName.from("a"), new LinkedHashMap<>()); + File testApp = new File("src/test/resources/deploy/app"); + ApplicationId app = ApplicationId.from(TenantName.from("a"), + ApplicationName.from("foo"), InstanceName.defaultName()); + models.get(app.tenant()).put(app, new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app)); + handler = new SuperModelController(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } - + @Test - public void test_super_model_reload() throws IOException, SAXException { - TenantName tenantA = TenantName.from("a"); - assertNotNull(controller.getHandler()); - long gen = counter.increment(); - controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); - assertNotNull(controller.getHandler()); - assertThat(controller.getHandler().getGeneration(), is(gen)); - controller.reloadConfig(tenantA, createApp(tenantA, "foo", 4l, 2)); - assertThat(controller.getHandler().getGeneration(), is(gen)); - // Test that a new app is used when there already exist an application with the same id - ApplicationId appId = new ApplicationId.Builder().tenant(tenantA).applicationName("foo").build(); - assertThat(((TestApplication) controller.getHandler().getSuperModel().getCurrentModels().get(tenantA).get(appId)).version, is(2l)); - gen = counter.increment(); - controller.reloadConfig(tenantA, createApp(tenantA, "bar", 2l, 3)); - assertThat(controller.getHandler().getGeneration(), is(gen)); + public void test_lb_config_simple() { + LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); + handler.getSuperModel().getConfig(lb); + LbServicesConfig lbc = new LbServicesConfig(lb); + assertThat(lbc.tenants().size(), is(1)); + assertThat(lbc.tenants("a").applications().size(), is(1)); + Applications app = lbc.tenants("a").applications("foo:prod:default:default"); + assertTrue(app.hosts().size() > 0); } - @Test - public void test_super_model_remove() throws IOException, SAXException { - TenantName tenantA = TenantName.from("a"); - TenantName tenantB = TenantName.from("b"); - long gen = counter.increment(); - controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); - controller.reloadConfig(tenantA, createApp(tenantA, "bar", 30l, 2)); - controller.reloadConfig(tenantB, createApp(tenantB, "baz", 9l, 3)); - assertThat(controller.getHandler().getGeneration(), is(gen)); - assertThat(controller.getHandler().getSuperModel().getCurrentModels().size(), is(2)); - assertThat(controller.getHandler().getSuperModel().getCurrentModels().get(TenantName.from("a")).size(), is(2)); - controller.removeApplication( - new ApplicationId.Builder().tenant("a").applicationName("unknown").build()); - assertThat(controller.getHandler().getGeneration(), is(gen)); - assertThat(controller.getHandler().getSuperModel().getCurrentModels().size(), is(2)); - assertThat(controller.getHandler().getSuperModel().getCurrentModels().get(TenantName.from("a")).size(), is(2)); - gen = counter.increment(); - controller.removeApplication( - new ApplicationId.Builder().tenant("a").applicationName("bar").build()); - assertThat(controller.getHandler().getSuperModel().getCurrentModels().size(), is(2)); - assertThat(controller.getHandler().getSuperModel().getCurrentModels().get(TenantName.from("a")).size(), is(1)); - assertThat(controller.getHandler().getGeneration(), is(gen)); - } - @Test - public void test_super_model_master_generation() throws IOException, SAXException { - TenantName tenantA = TenantName.from("a"); - long masterGen = 10; - controller = new SuperModelController(counter, - new TestConfigDefinitionRepo(), - new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen))); - - long gen = counter.increment(); - controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3L, 1)); - assertThat(controller.getHandler().getGeneration(), is(masterGen + gen)); + @Test(expected = UnknownConfigDefinitionException.class) + public void test_unknown_config_definition() { + String md5 = "asdfasf"; + Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", md5, null), DefContent.fromList(Collections.emptyList()), + "fromHost", md5, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED, + Optional.empty()) + .getRequest(); + JRTServerConfigRequestV3 v3Request = JRTServerConfigRequestV3.createFromRequest(request); + handler.resolveConfig(v3Request); } @Test - public void test_super_model_has_application_when_enabled() { - assertFalse(controller.hasApplication(ApplicationId.global(), Optional.empty())); - controller.enable(); - assertTrue(controller.hasApplication(ApplicationId.global(), Optional.empty())); + public void test_lb_config_multiple_apps() throws IOException, SAXException { + Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); + models.put(TenantName.from("t1"), new LinkedHashMap<>()); + models.put(TenantName.from("t2"), new LinkedHashMap<>()); + File testApp1 = new File("src/test/resources/deploy/app"); + File testApp2 = new File("src/test/resources/deploy/advancedapp"); + File testApp3 = new File("src/test/resources/deploy/advancedapp"); + // TODO must fix equals, hashCode on Tenant + Version vespaVersion = Version.fromIntValues(1, 2, 3); + models.get(TenantName.from("t1")).put(applicationId("mysimpleapp"), + new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp1)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("mysimpleapp"))); + models.get(TenantName.from("t1")).put(applicationId("myadvancedapp"), + new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp2)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("myadvancedapp"))); + models.get(TenantName.from("t2")).put(applicationId("minetooadvancedapp"), + new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp3)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("minetooadvancedapp"))); + + SuperModelController han = new SuperModelController(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); + han.getSuperModel().getConfig(lb); + LbServicesConfig lbc = new LbServicesConfig(lb); + assertThat(lbc.tenants().size(), is(2)); + assertThat(lbc.tenants("t1").applications().size(), is(2)); + assertThat(lbc.tenants("t2").applications().size(), is(1)); + assertThat(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default").hosts().size(), is(1)); + assertQrServer(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default")); } - private ApplicationSet createApp(TenantName tenant, String application, long generation, long version) throws IOException, SAXException { - return ApplicationSet.fromSingle( - new TestApplication( - new VespaModel(FilesApplicationPackage.fromFile(testApp)), - new ServerCache(), - generation, - new ApplicationId.Builder().tenant(tenant).applicationName(application).build(), - version)); + private ApplicationId applicationId(String applicationName) { + return ApplicationId.from(TenantName.defaultName(), + ApplicationName.from(applicationName), InstanceName.defaultName()); } - private static class TestApplication extends Application { - private long version = 0; - - public TestApplication(VespaModel vespaModel, ServerCache cache, long appGeneration, ApplicationId app, long version) { - super(vespaModel, cache, appGeneration, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app); - this.version = version; + private void assertQrServer(Applications app) { + String host = app.hosts().keySet().iterator().next(); + Applications.Hosts hosts = app.hosts(host); + assertThat(hosts.hostname(), is(host)); + for (Map.Entry<String, Applications.Hosts.Services> e : app.hosts(host).services().entrySet()) { + System.out.println(e); + if ("qrserver".equals(e.getKey())) { + Applications.Hosts.Services s = e.getValue(); + assertThat(s.type(), is("qrserver")); + assertThat(s.ports().size(), is(4)); + assertThat(s.index(), is(0)); + return; + } } + org.junit.Assert.fail("No qrserver service in config"); } } + + + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 65d6611112c..a2b0b4e7d22 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -1,37 +1,29 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.*; -import com.yahoo.jrt.Request; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications; -import com.yahoo.vespa.config.protocol.CompressionType; -import com.yahoo.vespa.config.protocol.DefContent; -import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; -import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; -import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.model.SuperModel; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; +import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; import java.io.File; import java.io.IOException; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Optional; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; /** * @author lulf @@ -39,93 +31,100 @@ import static org.junit.Assert.assertTrue; */ public class SuperModelRequestHandlerTest { - private SuperModelRequestHandler handler; + private static final File testApp = new File("src/test/resources/deploy/app"); + private SuperModelGenerationCounter counter; + private SuperModelRequestHandler controller; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); @Before - public void setupHandler() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); - models.put(TenantName.from("a"), new LinkedHashMap<>()); - File testApp = new File("src/test/resources/deploy/app"); - ApplicationId app = ApplicationId.from(TenantName.from("a"), - ApplicationName.from("foo"), InstanceName.defaultName()); - models.get(app.tenant()).put(app, new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), 4l, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app)); - handler = new SuperModelRequestHandler(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); + public void setup() throws IOException { + counter = new SuperModelGenerationCounter(new MockCurator()); + controller = new SuperModelRequestHandler(counter, + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder())); } - + @Test - public void test_lb_config_simple() { - LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); - handler.getSuperModel().getConfig(lb); - LbServicesConfig lbc = new LbServicesConfig(lb); - assertThat(lbc.tenants().size(), is(1)); - assertThat(lbc.tenants("a").applications().size(), is(1)); - Applications app = lbc.tenants("a").applications("foo:prod:default:default"); - assertTrue(app.hosts().size() > 0); + public void test_super_model_reload() throws IOException, SAXException { + TenantName tenantA = TenantName.from("a"); + assertNotNull(controller.getHandler()); + long gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + assertNotNull(controller.getHandler()); + assertThat(controller.getHandler().getGeneration(), is(gen)); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 4l, 2)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + // Test that a new app is used when there already exist an application with the same id + ApplicationId appId = new ApplicationId.Builder().tenant(tenantA).applicationName("foo").build(); + assertThat(((TestApplication) controller.getHandler().getSuperModel().applicationModels().get(tenantA).get(appId)).version, is(2l)); + gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "bar", 2l, 3)); + assertThat(controller.getHandler().getGeneration(), is(gen)); } + @Test + public void test_super_model_remove() throws IOException, SAXException { + TenantName tenantA = TenantName.from("a"); + TenantName tenantB = TenantName.from("b"); + long gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3l, 1)); + controller.reloadConfig(tenantA, createApp(tenantA, "bar", 30l, 2)); + controller.reloadConfig(tenantB, createApp(tenantB, "baz", 9l, 3)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getSuperModel().applicationModels().size(), is(2)); + assertThat(controller.getHandler().getSuperModel().applicationModels().get(TenantName.from("a")).size(), is(2)); + controller.removeApplication( + new ApplicationId.Builder().tenant("a").applicationName("unknown").build()); + assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getSuperModel().applicationModels().size(), is(2)); + assertThat(controller.getHandler().getSuperModel().applicationModels().get(TenantName.from("a")).size(), is(2)); + gen = counter.increment(); + controller.removeApplication( + new ApplicationId.Builder().tenant("a").applicationName("bar").build()); + assertThat(controller.getHandler().getSuperModel().applicationModels().size(), is(2)); + assertThat(controller.getHandler().getSuperModel().applicationModels().get(TenantName.from("a")).size(), is(1)); + assertThat(controller.getHandler().getGeneration(), is(gen)); + } - @Test(expected = UnknownConfigDefinitionException.class) - public void test_unknown_config_definition() { - String md5 = "asdfasf"; - Request request = JRTClientConfigRequestV3.createWithParams(new ConfigKey<>("foo", "id", "bar", md5, null), DefContent.fromList(Collections.emptyList()), - "fromHost", md5, 1, 1, Trace.createDummy(), CompressionType.UNCOMPRESSED, - Optional.empty()) - .getRequest(); - JRTServerConfigRequestV3 v3Request = JRTServerConfigRequestV3.createFromRequest(request); - handler.resolveConfig(v3Request); + @Test + public void test_super_model_master_generation() throws IOException, SAXException { + TenantName tenantA = TenantName.from("a"); + long masterGen = 10; + controller = new SuperModelRequestHandler(counter, + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder().masterGeneration(masterGen))); + + long gen = counter.increment(); + controller.reloadConfig(tenantA, createApp(tenantA, "foo", 3L, 1)); + assertThat(controller.getHandler().getGeneration(), is(masterGen + gen)); } @Test - public void test_lb_config_multiple_apps() throws IOException, SAXException { - Map<TenantName, Map<ApplicationId, Application>> models = new LinkedHashMap<>(); - models.put(TenantName.from("t1"), new LinkedHashMap<>()); - models.put(TenantName.from("t2"), new LinkedHashMap<>()); - File testApp1 = new File("src/test/resources/deploy/app"); - File testApp2 = new File("src/test/resources/deploy/advancedapp"); - File testApp3 = new File("src/test/resources/deploy/advancedapp"); - // TODO must fix equals, hashCode on Tenant - Version vespaVersion = Version.fromIntValues(1, 2, 3); - models.get(TenantName.from("t1")).put(applicationId("mysimpleapp"), - new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp1)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("mysimpleapp"))); - models.get(TenantName.from("t1")).put(applicationId("myadvancedapp"), - new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp2)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("myadvancedapp"))); - models.get(TenantName.from("t2")).put(applicationId("minetooadvancedapp"), - new Application(new VespaModel(FilesApplicationPackage.fromFile(testApp3)), new ServerCache(), 4l, vespaVersion, MetricUpdater.createTestUpdater(), applicationId("minetooadvancedapp"))); - - SuperModelRequestHandler han = new SuperModelRequestHandler(new SuperModel(models, Zone.defaultZone()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); - LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); - han.getSuperModel().getConfig(lb); - LbServicesConfig lbc = new LbServicesConfig(lb); - assertThat(lbc.tenants().size(), is(2)); - assertThat(lbc.tenants("t1").applications().size(), is(2)); - assertThat(lbc.tenants("t2").applications().size(), is(1)); - assertThat(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default").hosts().size(), is(1)); - assertQrServer(lbc.tenants("t2").applications("minetooadvancedapp:prod:default:default")); + public void test_super_model_has_application_when_enabled() { + assertFalse(controller.hasApplication(ApplicationId.global(), Optional.empty())); + controller.enable(); + assertTrue(controller.hasApplication(ApplicationId.global(), Optional.empty())); } - private ApplicationId applicationId(String applicationName) { - return ApplicationId.from(TenantName.defaultName(), - ApplicationName.from(applicationName), InstanceName.defaultName()); + private ApplicationSet createApp(TenantName tenant, String application, long generation, long version) throws IOException, SAXException { + return ApplicationSet.fromSingle( + new TestApplication( + new VespaModel(FilesApplicationPackage.fromFile(testApp)), + new ServerCache(), + generation, + new ApplicationId.Builder().tenant(tenant).applicationName(application).build(), + version)); } - private void assertQrServer(Applications app) { - String host = app.hosts().keySet().iterator().next(); - Applications.Hosts hosts = app.hosts(host); - assertThat(hosts.hostname(), is(host)); - for (Map.Entry<String, Applications.Hosts.Services> e : app.hosts(host).services().entrySet()) { - System.out.println(e); - if ("qrserver".equals(e.getKey())) { - Applications.Hosts.Services s = e.getValue(); - assertThat(s.type(), is("qrserver")); - assertThat(s.ports().size(), is(4)); - assertThat(s.index(), is(0)); - return; - } + private static class TestApplication extends Application { + private long version = 0; + + public TestApplication(VespaModel vespaModel, ServerCache cache, long appGeneration, ApplicationId app, long version) { + super(vespaModel, cache, appGeneration, Version.fromIntValues(1, 2, 3), MetricUpdater.createTestUpdater(), app); + this.version = version; } - org.junit.Assert.fail("No qrserver service in config"); } } - - - diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 30a3eec47fd..3c19725c22f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -26,8 +26,9 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.modelfactory.ModelResult; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; @@ -35,13 +36,10 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.Tenants; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; -import org.apache.curator.framework.CuratorFramework; -import org.junit.Before; import java.io.File; import java.io.IOException; @@ -112,11 +110,14 @@ public class DeployTester { } public Optional<com.yahoo.config.provision.Deployment> redeployFromLocalActive(ApplicationId id) { - ApplicationRepository applicationRepository = new ApplicationRepository(tenants, HostProvisionerProvider.withProvisioner(createHostProvisioner()), - new ConfigserverConfig(new ConfigserverConfig.Builder()), curator); - - Optional<com.yahoo.config.provision.Deployment> deployment = applicationRepository.deployFromLocalActive(id, Duration.ofSeconds(60)); - return deployment; + ApplicationRepository applicationRepository = new ApplicationRepository(tenants, + HostProvisionerProvider.withProvisioner(createHostProvisioner()), + new ConfigserverConfig(new ConfigserverConfig.Builder()), + curator, + new LogServerLogGrabber(), + new ApplicationConvergenceChecker()); + + return applicationRepository.deployFromLocalActive(id, Duration.ofSeconds(60)); } private Provisioner createHostProvisioner() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index f92bfa7d866..fc5a672559d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -9,9 +10,13 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Response; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.Session; +import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; import org.junit.Test; @@ -51,7 +56,17 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { testTenantBuilder.tenants().get(tenant2).getLocalSessionRepo().addSession(new MockSession(3l, FilesApplicationPackage.fromFile(new File("src/test/apps/content2")))); testTenantBuilder.tenants().get(tenant1).getApplicationRepo().createPutApplicationTransaction(idTenant1, 2l).commit(); testTenantBuilder.tenants().get(tenant2).getApplicationRepo().createPutApplicationTransaction(idTenant2, 3l).commit(); - handler = new ApplicationHandler(command -> command.run(), AccessLog.voidAccessLog(), testTenantBuilder.createTenants(), HostProvisionerProvider.empty(), Zone.defaultZone(), null, null, null, null); + handler = new ApplicationHandler(command -> command.run(), + AccessLog.voidAccessLog(), + testTenantBuilder.createTenants(), + HostProvisionerProvider.empty(), + Zone.defaultZone(), + new ApplicationRepository(testTenantBuilder.createTenants(), + HostProvisionerProvider.empty(), + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new MockCurator(), + new LogServerLogGrabber(), + new ApplicationConvergenceChecker())); pathPrefix = createPath(idTenant1, Zone.defaultZone()); baseUrl = baseServer + pathPrefix; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index a1a4a3413ba..f8071721989 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.FilesApplicationPackage; @@ -15,6 +16,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; import com.yahoo.jdisc.Response; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.SuperModelGenerationCounter; @@ -98,10 +100,12 @@ public class ApplicationHandlerTest { tenants, HostProvisionerProvider.withProvisioner(provisioner), Zone.defaultZone(), - convergeChecker, - logServerLogGrabber, - null, - null); + new ApplicationRepository(tenants, + HostProvisionerProvider.withProvisioner(provisioner), + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new MockCurator(), + logServerLogGrabber, + convergeChecker)); } private ApplicationHandler createApplicationHandler(Tenants tenants) { @@ -111,10 +115,12 @@ public class ApplicationHandlerTest { tenants, HostProvisionerProvider.withProvisioner(provisioner), Zone.defaultZone(), - new ApplicationConvergenceChecker(stateApiFactory), - new LogServerLogGrabber(), - null, - null); + new ApplicationRepository(tenants, + HostProvisionerProvider.withProvisioner(provisioner), + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new MockCurator(), + new LogServerLogGrabber(), + new ApplicationConvergenceChecker(stateApiFactory))); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 0b0c4ad0629..e03282da72b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -5,8 +5,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Collection; import java.util.List; -import java.util.concurrent.Executor; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.*; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.logging.AccessLog; @@ -15,6 +15,8 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.JsonFormat; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.*; +import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; +import com.yahoo.vespa.config.server.application.LogServerLogGrabber; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; @@ -158,13 +160,17 @@ public class SessionActiveHandlerTest extends SessionActiveHandlerTestBase { .withRemoteSessionRepo(remoteSessionRepo) .withApplicationRepo(applicationRepo) .build(); - return new SessionActiveHandler(new Executor() { - @SuppressWarnings("NullableProblems") - @Override - public void execute(Runnable command) { - command.run(); - } - }, AccessLog.voidAccessLog(), testTenantBuilder.createTenants(), HostProvisionerProvider.withProvisioner(hostProvisioner), Zone.defaultZone()); + return new SessionActiveHandler( + Runnable::run, + AccessLog.voidAccessLog(), + testTenantBuilder.createTenants(), + Zone.defaultZone(), + new ApplicationRepository(testTenantBuilder.createTenants(), + HostProvisionerProvider.withProvisioner(hostProvisioner), + new ConfigserverConfig(new ConfigserverConfig.Builder()), + curator, + new LogServerLogGrabber(), + new ApplicationConvergenceChecker())); } public static class MockProvisioner implements Provisioner { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java index 3831f94a77d..b7af5e09f47 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals; * @author lulf */ public class StaticProvisionerTest { + @Test public void sameHostsAreProvisioned() throws IOException, SAXException { ApplicationPackage app = FilesApplicationPackage.fromFile(new File("src/test/apps/hosted")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index 887d2b2f5d6..8f1754357b2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -11,11 +11,11 @@ import com.yahoo.jrt.Transport; import com.yahoo.net.HostName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.GenerationCounter; +import com.yahoo.vespa.config.server.SuperModelRequestHandler; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.MemoryGenerationCounter; import com.yahoo.vespa.config.server.PortRangeAllocator; -import com.yahoo.vespa.config.server.SuperModelController; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; @@ -82,9 +82,9 @@ public class TestWithRpc { protected void createAndStartRpcServer(boolean hostedVespa) { rpcServer = new RpcServer(new ConfigserverConfig(new ConfigserverConfig.Builder().rpcport(port).numthreads(1).maxgetconfigclients(1).hostedVespa(hostedVespa)), - new SuperModelController(generationCounter, - new TestConfigDefinitionRepo(), - new ConfigserverConfig(new ConfigserverConfig.Builder())), + new SuperModelRequestHandler(generationCounter, + new TestConfigDefinitionRepo(), + new ConfigserverConfig(new ConfigserverConfig.Builder())), Metrics.createTestMetrics(), new HostRegistries(), hostLivenessTracker); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); |