diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2019-07-11 13:45:17 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@verizonmedia.com> | 2019-07-11 13:45:17 +0200 |
commit | 9549a8005480a3fe61fb087e359a4a442180819f (patch) | |
tree | e76ded2ddf9a8127cd780304105313c22640c8dc /configserver | |
parent | 9f3a3b9f6962bd20714b99a046860de1886be600 (diff) | |
parent | 1c79079945c56fa91de8427fbc8f2170eec9ed8c (diff) |
Merge remote-tracking branch 'origin/master' into olaa/cfg-server-metric-aggregation
Conflicts:
configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
Diffstat (limited to 'configserver')
107 files changed, 2273 insertions, 675 deletions
diff --git a/configserver/CMakeLists.txt b/configserver/CMakeLists.txt index 4c98ca64847..8f939e54d55 100644 --- a/configserver/CMakeLists.txt +++ b/configserver/CMakeLists.txt @@ -2,7 +2,6 @@ install_configserver_component(configserver) vespa_install_script(src/main/sh/vespa-configserver-remove-state bin) -vespa_install_script(src/main/sh/ping-configserver libexec/vespa) vespa_install_script(src/main/sh/start-configserver libexec/vespa) vespa_install_script(src/main/sh/start-logd libexec/vespa) vespa_install_script(src/main/sh/stop-configserver libexec/vespa) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 359650df514..020e0a69205 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -29,6 +29,7 @@ import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.FileDistributionStatus; import com.yahoo.vespa.config.server.application.HttpProxy; @@ -39,7 +40,6 @@ import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.Deployment; import com.yahoo.vespa.config.server.deploy.InfraDeployerProvider; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; import com.yahoo.vespa.config.server.http.v2.PrepareResult; @@ -86,6 +86,7 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERC import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; +import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; import static java.nio.file.Files.readAttributes; /** @@ -109,7 +110,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final ConfigserverConfig configserverConfig; private final FileDistributionStatus fileDistributionStatus; private final Orchestrator orchestrator; - private final LogRetriever logRetriever = new LogRetriever(); + private final LogRetriever logRetriever; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -119,9 +120,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye HttpProxy httpProxy, ConfigserverConfig configserverConfig, Orchestrator orchestrator) { - this(tenantRepository, hostProvisionerProvider.getHostProvisioner(), infraDeployerProvider.getInfraDeployer(), - configConvergenceChecker, httpProxy, configserverConfig, orchestrator, - Clock.systemUTC(), new FileDistributionStatus()); + this(tenantRepository, + hostProvisionerProvider.getHostProvisioner(), + infraDeployerProvider.getInfraDeployer(), + configConvergenceChecker, + httpProxy, + configserverConfig, + orchestrator, + new LogRetriever(), + new FileDistributionStatus(), + Clock.systemUTC()); } // For testing @@ -129,17 +137,45 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Provisioner hostProvisioner, Orchestrator orchestrator, Clock clock) { - this(tenantRepository, hostProvisioner, orchestrator, clock, new ConfigserverConfig(new ConfigserverConfig.Builder())); + this(tenantRepository, + hostProvisioner, + orchestrator, + new ConfigserverConfig(new ConfigserverConfig.Builder()), + new LogRetriever(), + clock); } // For testing public ApplicationRepository(TenantRepository tenantRepository, Provisioner hostProvisioner, Orchestrator orchestrator, - Clock clock, - ConfigserverConfig configserverConfig) { - this(tenantRepository, Optional.of(hostProvisioner), Optional.empty(), new ConfigConvergenceChecker(), new HttpProxy(new SimpleHttpFetcher()), - configserverConfig, orchestrator, clock, new FileDistributionStatus()); + LogRetriever logRetriever, + Clock clock) { + this(tenantRepository, + hostProvisioner, + orchestrator, + new ConfigserverConfig(new ConfigserverConfig.Builder()), + logRetriever, + clock); + } + + // For testing + public ApplicationRepository(TenantRepository tenantRepository, + Provisioner hostProvisioner, + Orchestrator orchestrator, + ConfigserverConfig configserverConfig, + LogRetriever logRetriever, + Clock clock) { + this(tenantRepository, + Optional.of(hostProvisioner), + Optional.empty(), + new ConfigConvergenceChecker(), + new HttpProxy(new SimpleHttpFetcher()), + configserverConfig, + orchestrator, + logRetriever, + new FileDistributionStatus(), + clock); } private ApplicationRepository(TenantRepository tenantRepository, @@ -149,17 +185,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye HttpProxy httpProxy, ConfigserverConfig configserverConfig, Orchestrator orchestrator, - Clock clock, - FileDistributionStatus fileDistributionStatus) { + LogRetriever logRetriever, + FileDistributionStatus fileDistributionStatus, + Clock clock) { this.tenantRepository = tenantRepository; this.hostProvisioner = hostProvisioner; this.infraDeployer = infraDeployer; this.convergeChecker = configConvergenceChecker; this.httpProxy = httpProxy; - this.clock = clock; this.configserverConfig = configserverConfig; this.orchestrator = orchestrator; + this.logRetriever = logRetriever; this.fileDistributionStatus = fileDistributionStatus; + this.clock = clock; } // ---------------- Deploying ---------------------------------------------------------------- @@ -318,22 +356,24 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye TenantApplications tenantApplications = tenant.getApplicationRepo(); try (Lock lock = tenantApplications.lock(applicationId)) { if ( ! tenantApplications.exists(applicationId)) return false; + // Deleting an application is done by deleting the remote session and waiting // until the config server where the deployment happened picks it up and deletes // the local session - long sessionId = tenantApplications.requireActiveSessionOf(applicationId); - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - // TODO: Add support for timeout in request - Duration waitTime = Duration.ofSeconds(60); - if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - } else { - log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); - return false; - } + boolean sessionDeleted = tenantApplications.activeSessionOf(applicationId).map(sessionId -> { + RemoteSession remoteSession = getRemoteSession(tenant, sessionId); + remoteSession.createDeleteTransaction().commit(); + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + // TODO: Add support for timeout in request + Duration waitTime = Duration.ofSeconds(60); + if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + return true; + } else { + log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + return false; + } + }).orElse(true); NestedTransaction transaction = new NestedTransaction(); transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested @@ -344,9 +384,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); transaction.commit(); + return sessionDeleted; } - - return true; } public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) { @@ -419,12 +458,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private Application getApplication(ApplicationId applicationId) { + return getApplication(applicationId, Optional.empty()); + } + + private Application getApplication(ApplicationId applicationId, Optional<Version> version) { try { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); - if (tenant == null) throw new IllegalArgumentException("Tenant '" + applicationId.tenant() + "' not found"); + if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0); - return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty(), clock.instant()); + return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); + } catch (NotFoundException e) { + log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); + throw e; } catch (Exception e) { log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "'", e); throw e; @@ -468,12 +514,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Convergence ---------------------------------------------------------------- - public HttpResponse checkServiceForConfigConvergence(ApplicationId applicationId, String hostAndPort, URI uri, Duration timeout) { - return convergeChecker.checkService(getApplication(applicationId), hostAndPort, uri, timeout); + public HttpResponse checkServiceForConfigConvergence(ApplicationId applicationId, String hostAndPort, URI uri, + Duration timeout, Optional<Version> vespaVersion) { + return convergeChecker.checkService(getApplication(applicationId, vespaVersion), hostAndPort, uri, timeout); } - public HttpResponse servicesToCheckForConfigConvergence(ApplicationId applicationId, URI uri, Duration timeoutPerService) { - return convergeChecker.servicesToCheck(getApplication(applicationId), uri, timeoutPerService); + public HttpResponse servicesToCheckForConfigConvergence(ApplicationId applicationId, URI uri, + Duration timeoutPerService, Optional<Version> vespaVersion) { + return convergeChecker.servicesToCheck(getApplication(applicationId, vespaVersion), uri, timeoutPerService); } // ---------------- Logs ---------------------------------------------------------------- @@ -494,10 +542,16 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getActiveSession(tenantRepository.getTenant(applicationId.tenant()), applicationId); } - public long getSessionIdForApplication(Tenant tenant, ApplicationId applicationId) { + public long getSessionIdForApplication(ApplicationId applicationId) { + Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); + return getSessionIdForApplication(tenant, applicationId); + } + + private long getSessionIdForApplication(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); if (applicationRepo == null) - throw new IllegalArgumentException("Application repo for tenant '" + tenant.getName() + "' not found"); + throw new NotFoundException("Application repo for tenant '" + tenant.getName() + "' not found"); return applicationRepo.requireActiveSessionOf(applicationId); } @@ -567,7 +621,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return tenantRepository.getAllTenantNames().stream() .filter(tenantName -> activeApplications(tenantName).isEmpty()) .filter(tenantName -> !tenantName.equals(TenantName.defaultName())) // Not allowed to remove 'default' tenant - .filter(tenantName -> !tenantName.equals(TenantRepository.HOSTED_VESPA_TENANT)) // Not allowed to remove 'hosted-vespa' tenant + .filter(tenantName -> !tenantName.equals(HOSTED_VESPA_TENANT)) // Not allowed to remove 'hosted-vespa' tenant .filter(tenantName -> tenantRepository.getTenant(tenantName).getCreatedTime().isBefore(now.minus(ttlForUnusedTenant))) .peek(tenantRepository::deleteTenant) .collect(Collectors.toSet()); @@ -595,19 +649,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Misc operations ---------------------------------------------------------------- - public Tenant verifyTenantAndApplication(ApplicationId applicationId) { - TenantName tenantName = applicationId.tenant(); - if (!tenantRepository.checkThatTenantExists(tenantName)) { - throw new IllegalArgumentException("Tenant " + tenantName + " was not found."); - } - Tenant tenant = tenantRepository.getTenant(tenantName); - List<ApplicationId> applicationIds = listApplicationIds(tenant); - if (!applicationIds.contains(applicationId)) { - throw new IllegalArgumentException("No such application id: " + applicationId); - } - return tenant; - } - public ApplicationMetaData getMetadataFromSession(Tenant tenant, long sessionId) { return getLocalSession(tenant, sessionId).getMetaData(); } @@ -669,11 +710,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - private List<ApplicationId> listApplicationIds(Tenant tenant) { - TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.activeApplications(); - } - private void cleanupTempDirectory(File tempDir) { logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { @@ -710,22 +746,19 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private String getLogServerURI(ApplicationId applicationId, Optional<String> hostname) { - Application application = getApplication(applicationId); - Collection<HostInfo> hostInfos = application.getModel().getHosts(); - - // In ServiceInfo: node-admin does not have - // 1) Correct ports - // 2) logserver - // Assume if hostname is set that this is node-admin hostname - // TODO: Fix and simplify this once the above to problems have been fixed + // Allow to get logs from a given hostname if the application is under the hosted-vespa tenant. + // We make no validation that the hostname is actually allocated to the given application since + // most applications under hosted-vespa are not known to the model and its OK for a user to get + // logs for any host if they are authorized for the hosted-vespa tenant. if (hostname.isPresent()) { - HostInfo logServerHostInfo = hostInfos.stream() - .filter(host -> host.getHostname().equalsIgnoreCase(hostname.get())) - .findFirst().orElseThrow(() -> - new IllegalArgumentException("Host " + hostname.get() + " does not belong to " + applicationId)); - return "http://" + logServerHostInfo.getHostname() + ":8080/logs"; + if (HOSTED_VESPA_TENANT.equals(applicationId.tenant())) + return "http://" + hostname.get() + ":8080/logs"; + else throw new IllegalArgumentException("Only hostname paramater unsupported for application " + applicationId); } + Application application = getApplication(applicationId); + Collection<HostInfo> hostInfos = application.getModel().getHosts(); + HostInfo logServerHostInfo = hostInfos.stream() .filter(host -> host.getServices().stream() .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver"))) @@ -782,7 +815,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye static Version decideVersion(ApplicationId application, Environment environment, Version sessionVersion, boolean bootstrap) { if ( environment.isManuallyDeployed() && sessionVersion.getMajor() == Vtag.currentVersion.getMajor() - && ! "hosted-vespa".equals(application.tenant().value()) // Never change version of system applications + && ! HOSTED_VESPA_TENANT.equals(application.tenant()) // Never change version of system applications && ! application.instance().isTester() // Never upgrade tester containers && ! bootstrap) { // Do not use current version when bootstrapping config server return Vtag.currentVersion; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index d631cc18d75..d490b1b49e9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -7,13 +7,13 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployment; +import com.yahoo.config.provision.TransientException; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; +import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; @@ -30,6 +30,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import static com.yahoo.vespa.config.server.ConfigServerBootstrap.Mode.BOOTSTRAP_IN_CONSTRUCTOR; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApplicationsFails.CONTINUE; import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApplicationsFails.EXIT_JVM; @@ -69,13 +70,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @SuppressWarnings("unused") @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus, - FlagSource flagSource) { - this(applicationRepository, server, versionState, stateMonitor, vipStatus, - Flags.CONFIG_SERVER_BOOTSTRAP_IN_SEPARATE_THREAD.bindTo(flagSource).value() - ? Mode.BOOTSTRAP_IN_SEPARATE_THREAD - : Mode.BOOTSTRAP_IN_CONSTRUCTOR, - EXIT_JVM, + VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) { + this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM, applicationRepository.configserverConfig().hostedVespa() ? VipStatusMode.VIP_STATUS_FILE : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY); @@ -247,10 +243,13 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) { + ApplicationId app = f.getKey(); try { f.getValue().get(); + } catch (TransientException e) { + log.log(LogLevel.INFO, "Redeploying " + app + + " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e)); } catch (ExecutionException e) { - ApplicationId app = f.getKey(); log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e); failedDeployments.add(app); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index ad12c5c9e24..1eb18773898 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -2,9 +2,12 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; @@ -17,6 +20,7 @@ import com.yahoo.vespa.flags.FlagSource; import java.time.Clock; import java.util.Optional; +import java.util.concurrent.ExecutorService; /** * Interface representing all global config server components used within the config server. @@ -32,7 +36,6 @@ public interface GlobalComponentRegistry { ConfigserverConfig getConfigserverConfig(); TenantListener getTenantListener(); ReloadListener getReloadListener(); - SuperModelGenerationCounter getSuperModelGenerationCounter(); ConfigDefinitionRepo getStaticConfigDefinitionRepo(); PermanentApplicationPackage getPermanentApplicationPackage(); HostRegistries getHostRegistries(); @@ -41,6 +44,8 @@ public interface GlobalComponentRegistry { Zone getZone(); Clock getClock(); ConfigServerDB getConfigServerDB(); + StripedExecutor<TenantName> getZkWatcherExecutor(); FlagSource getFlagSource(); - + ExecutorService getZkCacheExecutor(); + SecretStore getSecretStore(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index e9ebe954799..9badd19009f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -3,9 +3,13 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; @@ -14,12 +18,15 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.TenantListener; +import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import java.time.Clock; import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * Registry containing all the "static"/"global" components in a config server in one place. @@ -35,7 +42,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry private final SessionPreparer sessionPreparer; private final RpcServer rpcServer; private final ConfigserverConfig configserverConfig; - private final SuperModelGenerationCounter superModelGenerationCounter; private final ConfigDefinitionRepo staticConfigDefinitionRepo; private final PermanentApplicationPackage permanentApplicationPackage; private final HostRegistries hostRegistries; @@ -43,6 +49,9 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry private final Zone zone; private final ConfigServerDB configServerDB; private final FlagSource flagSource; + private final SecretStore secretStore; + private final StripedExecutor<TenantName> zkWatcherExecutor; + private final ExecutorService zkCacheExecutor; @SuppressWarnings("WeakerAccess") @Inject @@ -60,7 +69,8 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry HostProvisionerProvider hostProvisionerProvider, Zone zone, ConfigServerDB configServerDB, - FlagSource flagSource) { + FlagSource flagSource, + SecretStore secretStore) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; @@ -68,7 +78,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry this.sessionPreparer = sessionPreparer; this.rpcServer = rpcServer; this.configserverConfig = configserverConfig; - this.superModelGenerationCounter = superModelGenerationCounter; this.staticConfigDefinitionRepo = staticConfigDefinitionRepo; this.permanentApplicationPackage = permanentApplicationPackage; this.hostRegistries = hostRegistries; @@ -76,6 +85,9 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry this.zone = zone; this.configServerDB = configServerDB; this.flagSource = flagSource; + this.secretStore = secretStore; + this.zkWatcherExecutor = new StripedExecutor<>(); + this.zkCacheExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(TenantRepository.class.getName())); } @Override @@ -93,8 +105,6 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry @Override public ReloadListener getReloadListener() { return rpcServer; } @Override - public SuperModelGenerationCounter getSuperModelGenerationCounter() { return superModelGenerationCounter; } - @Override public ConfigDefinitionRepo getStaticConfigDefinitionRepo() { return staticConfigDefinitionRepo; } @Override public PermanentApplicationPackage getPermanentApplicationPackage() { return permanentApplicationPackage; } @@ -120,5 +130,20 @@ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry public ConfigServerDB getConfigServerDB() { return configServerDB; } @Override + public StripedExecutor<TenantName> getZkWatcherExecutor() { + return zkWatcherExecutor; + } + + @Override public FlagSource getFlagSource() { return flagSource; } + + @Override + public ExecutorService getZkCacheExecutor() { + return zkCacheExecutor; + } + + @Override + public SecretStore getSecretStore() { + return secretStore; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java index 185958510b6..c3d9fb15212 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/RequestHandler.java @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import java.util.Optional; -import java.util.Set; - import com.yahoo.component.Version; +import com.yahoo.config.FileReference; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.protocol.ConfigResponse; +import java.util.Optional; +import java.util.Set; + /** * Instances of this can serve misc config related requests * @@ -83,4 +84,12 @@ public interface RequestHandler { * @return an {@link ApplicationId} instance. */ ApplicationId resolveApplicationId(String hostName); + + /** + * Returns the set of file references from the application's Vespa models, aggregated across all application versions. + * + * @param applicationId application id to use + * @return set of file references that is owned by the application + */ + Set<FileReference> listFileReferences(ApplicationId applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java index c2c17cd7b78..6adbcc8dae9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java @@ -40,53 +40,4 @@ public class SuperModelGenerationCounter implements GenerationCounter { return counter.get(); } - /** Returns a transaction which increments this */ - public IncrementTransaction incrementTransaction() { - return new IncrementTransaction(counter); - } - - /** An increment transaction */ - public static class IncrementTransaction extends AbstractTransaction { - - /** Creates a counting curator transaction containing a single increment operation */ - public IncrementTransaction(CuratorCounter counter) { - add(new IncrementOperation(counter)); - } - - @Override - public void prepare() { } - - @Override - public void commit() { - for (Operation operation : operations()) - ((IncrementOperation)operation).commit(); - } - - @Override - public void rollbackOrLog() { - for (Operation operation : operations()) - ((IncrementOperation)operation).rollback(); - } - - public static class IncrementOperation implements Transaction.Operation { - - private final CuratorCounter counter; - - public IncrementOperation(CuratorCounter counter) { - this.counter = counter; - } - - public void commit() { - counter.next(); - } - - public void rollback() { - // ok; we're just losing a generation number - } - - public String toString() { return "increment " + counterPath + " operation"; } - - } - - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index eeae770da43..ea835206b7c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -92,6 +92,7 @@ public class SuperModelManager implements SuperModelProvider { SuperModel newSuperModel = this.superModelConfigProvider .getSuperModel() .cloneAndSetApplication(applicationInfo); + generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); listeners.stream().forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo)); @@ -103,6 +104,7 @@ public class SuperModelManager implements SuperModelProvider { SuperModel newSuperModel = this.superModelConfigProvider .getSuperModel() .cloneAndRemoveApplication(applicationId); + generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); listeners.stream().forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index d77ca396974..520972f2fcf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.component.Version; import com.yahoo.log.LogLevel; @@ -117,6 +118,11 @@ public class SuperModelRequestHandler implements RequestHandler { return ApplicationId.global(); } + @Override + public Set<FileReference> listFileReferences(ApplicationId applicationId) { + throw new UnsupportedOperationException(); + } + public void enable() { enabled = true; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java index 4fe2a018766..5ce9ebca69d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java @@ -1,18 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.NotFoundException; import java.time.Instant; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import com.yahoo.vespa.config.server.NotFoundException; - /** * Used during config request handling to route to the right config model * based on application id and version. @@ -81,4 +81,8 @@ public final class ApplicationMapper { return Collections.unmodifiableSet(requestHandlers.keySet()); } + public List<Application> listApplications(ApplicationId applicationId) { + return requestHandlers.get(applicationId).getAllApplications(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java index 3444ffc2865..41119077b28 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java @@ -92,4 +92,8 @@ public final class ApplicationSet { return generation; } + List<Application> getAllApplications() { + return new ArrayList<>(applications.values()); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 8619435389a..c94f739b958 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -1,9 +1,11 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; import com.google.common.io.Files; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.http.BadRequestException; +import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; @@ -18,7 +20,6 @@ import java.util.zip.GZIPInputStream; * A compressed application points to an application package that can be decompressed. * * @author Ulf Lilleengen - * @since 5.1 */ public class CompressedApplicationInputStream implements AutoCloseable { @@ -41,7 +42,7 @@ public class CompressedApplicationInputStream implements AutoCloseable { } } - public static CompressedApplicationInputStream createFromCompressedStream(ArchiveInputStream ais) { + static CompressedApplicationInputStream createFromCompressedStream(ArchiveInputStream ais) { return new CompressedApplicationInputStream(ais); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 4fbda42fdc7..d55e07540d6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -7,7 +7,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; import org.glassfish.jersey.client.ClientProperties; @@ -22,13 +22,12 @@ import javax.ws.rs.client.WebTarget; import java.net.URI; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; @@ -43,17 +42,17 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER */ public class ConfigConvergenceChecker extends AbstractComponent { - private static final ApplicationId routingApplicationId = ApplicationId.from("hosted-vespa", "routing", "default"); + private static final Logger log = Logger.getLogger(ConfigConvergenceChecker.class.getName()); private static final String statePath = "/state/v1/"; private static final String configSubPath = "config"; - private final static Set<String> serviceTypesToCheck = new HashSet<>(Arrays.asList( + private final static Set<String> serviceTypesToCheck = Set.of( CONTAINER.serviceName, QRSERVER.serviceName, LOGSERVER_CONTAINER.serviceName, "searchnode", "storagenode", "distributor" - )); + ); private final StateApiFactory stateApiFactory; @@ -68,11 +67,11 @@ public class ConfigConvergenceChecker extends AbstractComponent { /** Check all services in given application. Returns the minimum current generation of all services */ public ServiceListResponse servicesToCheck(Application application, URI requestUrl, Duration timeoutPerService) { + log.log(LogLevel.DEBUG, () -> "Finding services to check config convergence for in '" + application); List<ServiceInfo> servicesToCheck = new ArrayList<>(); application.getModel().getHosts() .forEach(host -> host.getServices().stream() .filter(service -> serviceTypesToCheck.contains(service.getServiceType())) - .filter(service -> ! isHostAdminService(application.getId(), service)) .forEach(service -> getStatePort(service).ifPresent(port -> servicesToCheck.add(service)))); Map<ServiceInfo, Long> currentGenerations = getServiceGenerations(servicesToCheck, timeoutPerService); @@ -176,13 +175,6 @@ public class ConfigConvergenceChecker extends AbstractComponent { return WebResourceFactory.newResource(StateApi.class, target); } - private static boolean isHostAdminService(ApplicationId id, ServiceInfo service) { - return routingApplicationId.equals(id) - && service.getProperty("clustername") - .map("node-admin"::equals) - .orElse(false); - } - private static class ServiceListResponse extends JSONResponse { // Pre-condition: servicesToCheck has a state port diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 4be87253ceb..5b2f6cab3c4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -8,6 +9,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; @@ -20,9 +23,11 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Logger; @@ -45,26 +50,27 @@ public class TenantApplications { private final Curator curator; private final Path applicationsPath; private final Path locksPath; - // One thread pool for all instances of this class - private static final ExecutorService pathChildrenExecutor = - Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(TenantApplications.class.getName())); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final Map<ApplicationId, Lock> locks; + private final Executor zkWatcherExecutor; - private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { + private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant, + ExecutorService zkCacheExecutor, StripedExecutor<TenantName> zkWatcherExecutor) { this.curator = curator; this.applicationsPath = TenantRepository.getApplicationsPath(tenant); this.locksPath = TenantRepository.getLocksPath(tenant); this.locks = new ConcurrentHashMap<>(2); this.reloadHandler = reloadHandler; - this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); + this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); + this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor); this.directoryCache.start(); this.directoryCache.addListener(this::childEvent); } - public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { - return new TenantApplications(curator, reloadHandler, tenant); + public static TenantApplications create(GlobalComponentRegistry registry, ReloadHandler reloadHandler, TenantName tenant) { + return new TenantApplications(registry.getCurator(), reloadHandler, tenant, + registry.getZkCacheExecutor(), registry.getZkWatcherExecutor()); } /** @@ -85,10 +91,10 @@ public class TenantApplications { } /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */ - private OptionalLong activeSessionOf(ApplicationId id) { + public Optional<Long> activeSessionOf(ApplicationId id) { String data = curator.getData(applicationPath(id)).map(Utf8::toString) - .orElseThrow(() -> new IllegalArgumentException("Unknown application '" + id + "'.")); - return data.isEmpty() ? OptionalLong.empty() : OptionalLong.of(Long.parseLong(data)); + .orElseThrow(() -> new NotFoundException("No such application id: '" + id + "'")); + return data.isEmpty() ? Optional.empty() : Optional.of(Long.parseLong(data)); } /** @@ -152,23 +158,25 @@ public class TenantApplications { } private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { - switch (event.getType()) { - case CHILD_ADDED: - applicationAdded(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); - break; - // Event CHILD_REMOVED will be triggered on all config servers if deleteApplication() above is called on one of them - case CHILD_REMOVED: - applicationRemoved(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); - break; - case CHILD_UPDATED: - // do nothing, application just got redeployed - break; - default: - break; - } - // We may have lost events and may need to remove applications. - // New applications are added when session is added, not here. See RemoteSessionRepo. - removeUnusedApplications(); + zkWatcherExecutor.execute(() -> { + switch (event.getType()) { + case CHILD_ADDED: + applicationAdded(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); + break; + // Event CHILD_REMOVED will be triggered on all config servers if deleteApplication() above is called on one of them + case CHILD_REMOVED: + applicationRemoved(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); + break; + case CHILD_UPDATED: + // do nothing, application just got redeployed + break; + default: + break; + } + // We may have lost events and may need to remove applications. + // New applications are added when session is added, not here. See RemoteSessionRepo. + removeUnusedApplications(); + }); } private void applicationRemoved(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java index d300fabd68d..d783dc105c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java @@ -11,7 +11,6 @@ import java.util.List; * The actions are split into restart and re-feed actions. * * @author geirst - * @since 5.44 */ public class ConfigChangeActions { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java index 37120871234..29b0b99e42e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -10,7 +10,6 @@ import java.util.*; * Represents all actions to restart services in order to handle a config change. * * @author geirst - * @since 5.44 */ public class RestartActions { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java index ac2e18c6301..f1e2a938393 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActionsFormatter.java @@ -5,7 +5,6 @@ package com.yahoo.vespa.config.server.configchange; * Class used to format restart actions for human readability. * * @author geirst - * @since 5.44 */ public class RestartActionsFormatter { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 0279d175488..d875385d14d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -7,9 +7,11 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Rotation; @@ -126,13 +128,14 @@ public class ModelContextImpl implements ModelContext { private final boolean hostedVespa; private final Zone zone; private final Set<Rotation> rotations; + private final Set<ContainerEndpoint> endpoints; private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useDedicatedNodeForLogserver; private final boolean useFdispatchByDefault; private final boolean useAdaptiveDispatch; private final boolean dispatchWithProtobuf; - private final boolean enableMetricsProxyContainer; + private final Optional<TlsSecrets> tlsSecrets; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -143,9 +146,11 @@ public class ModelContextImpl implements ModelContext { boolean hostedVespa, Zone zone, Set<Rotation> rotations, + Set<ContainerEndpoint> endpoints, boolean isBootstrap, boolean isFirstTimeDeployment, - FlagSource flagSource) { + FlagSource flagSource, + Optional<TlsSecrets> tlsSecrets) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -155,6 +160,7 @@ public class ModelContextImpl implements ModelContext { this.hostedVespa = hostedVespa; this.zone = zone; this.rotations = rotations; + this.endpoints = endpoints; this.isBootstrap = isBootstrap; this.isFirstTimeDeployment = isFirstTimeDeployment; this.useDedicatedNodeForLogserver = Flags.USE_DEDICATED_NODE_FOR_LOGSERVER.bindTo(flagSource) @@ -165,8 +171,7 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.enableMetricsProxyContainer = Flags.ENABLE_METRICS_PROXY_CONTAINER.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.tlsSecrets = tlsSecrets; } @Override @@ -201,6 +206,9 @@ public class ModelContextImpl implements ModelContext { public Set<Rotation> rotations() { return rotations; } @Override + public Set<ContainerEndpoint> endpoints() { return endpoints; } + + @Override public boolean isBootstrap() { return isBootstrap; } @Override @@ -219,7 +227,7 @@ public class ModelContextImpl implements ModelContext { public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; } @Override - public boolean enableMetricsProxyContainer() { return enableMetricsProxyContainer; } + public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index e5552de41f5..110e73bcdf9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; @@ -9,7 +10,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; +import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -361,7 +362,9 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), hosts.toJson()); + configCurator.putData( + rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map<Version, FileRegistry> fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index a29891ae764..3d2ecd4a2ca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -48,7 +48,8 @@ public class HttpErrorResponse extends HttpResponse { OUT_OF_CAPACITY, REQUEST_TIMEOUT, UNKNOWN_VESPA_VERSION, - PARENT_HOST_NOT_READY + PARENT_HOST_NOT_READY, + CERTIFICATE_NOT_READY } public static HttpErrorResponse notFoundError(String msg) { @@ -95,6 +96,10 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(CONFLICT, errorCodes.PARENT_HOST_NOT_READY.name(), msg); } + public static HttpErrorResponse certificateNotReady(String msg) { + return new HttpErrorResponse(CONFLICT, errorCodes.CERTIFICATE_NOT_READY.name(), msg); + } + @Override public void render(OutputStream stream) throws IOException { new JsonFormat(true).encode(stream, slime); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index cd2052653ed..20ee77be9fe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.provision.ApplicationLockException; +import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -64,6 +65,8 @@ public class HttpHandler extends LoggingRequestHandler { return HttpErrorResponse.applicationLockFailure(getMessage(e, request)); } catch (ParentHostUnavailableException e) { return HttpErrorResponse.parentHostNotReady(getMessage(e, request)); + } catch (CertificateNotReadyException e) { + return HttpErrorResponse.certificateNotReady(getMessage(e, request)); } catch (Exception e) { log.log(LogLevel.WARNING, "Unexpected exception handling a config server request", e); return HttpErrorResponse.internalServerError(getMessage(e, request)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index 3f7c870210f..791d596d398 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -12,6 +12,9 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Optional; +/** + * @author olaaun + */ public class LogRetriever { private final HttpClient httpClient = HttpClientBuilder.create().build(); @@ -26,6 +29,7 @@ public class LogRetriever { } private static class ProxyResponse extends HttpResponse { + private final org.apache.http.HttpResponse clientResponse; private ProxyResponse(org.apache.http.HttpResponse clientResponse) { @@ -45,4 +49,5 @@ public class LogRetriever { clientResponse.getEntity().writeTo(outputStream); } } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index 7f409e4c8fa..3caaa7693a9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; +import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import com.yahoo.vespa.config.server.session.PrepareParams; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 37f78dafa49..421030fa77b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -56,13 +57,13 @@ public class ApplicationHandler extends HttpHandler { @Override public HttpResponse handleGET(HttpRequest request) { ApplicationId applicationId = getApplicationIdFromRequest(request); - Tenant tenant = verifyTenantAndApplication(applicationId); Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); if (isServiceConvergeRequest(request)) { // Expects both hostname and port in the request (hostname:port) String hostAndPort = getHostNameFromRequest(request); - return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), timeout); + return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), + timeout, getVespaVersionFromRequest(request)); } if (isClusterControllerStatusRequest(request)) { @@ -72,10 +73,10 @@ public class ApplicationHandler extends HttpHandler { } if (isContentRequest(request)) { - long sessionId = applicationRepository.getSessionIdForApplication(tenant, applicationId); + long sessionId = applicationRepository.getSessionIdForApplication(applicationId); String contentPath = ApplicationContentRequest.getContentPath(request); ApplicationFile applicationFile = - applicationRepository.getApplicationFileFromSession(tenant.getName(), + applicationRepository.getApplicationFileFromSession(applicationId.tenant(), sessionId, contentPath, ContentRequest.getApplicationFileMode(request.getMethod())); @@ -89,7 +90,8 @@ public class ApplicationHandler extends HttpHandler { } if (isServiceConvergeListRequest(request)) { - return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), timeout); + return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), timeout, + getVespaVersionFromRequest(request)); } if (isFiledistributionStatusRequest(request)) { @@ -136,14 +138,6 @@ public class ApplicationHandler extends HttpHandler { request.getProperty("clusterId")); } - private Tenant verifyTenantAndApplication(ApplicationId applicationId) { - try { - return applicationRepository.verifyTenantAndApplication(applicationId); - } catch (IllegalArgumentException e) { - throw new NotFoundException(e.getMessage()); - } - } - private static BindingMatch<?> getBindingMatch(HttpRequest request) { return HttpConfigRequests.getBindingMatch(request, // WARNING: UPDATE src/main/resources/configserver-app/services.xml IF YOU MAKE ANY CHANGES TO THESE BINDINGS! @@ -235,6 +229,13 @@ public class ApplicationHandler extends HttpHandler { .build(); } + private static Optional<Version> getVespaVersionFromRequest(HttpRequest request) { + String vespaVersion = request.getProperty("vespaVersion"); + return (vespaVersion == null || vespaVersion.isEmpty()) + ? Optional.empty() + : Optional.of(Version.fromString(vespaVersion)); + } + private static class DeleteApplicationResponse extends JSONResponse { DeleteApplicationResponse(int status, ApplicationId applicationId) { super(status); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 505ae38161d..e2495de18ba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -53,7 +53,7 @@ public class ConfigServerMaintenance extends AbstractComponent { DefaultTimes(ConfigserverConfig configserverConfig) { this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); - boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); + boolean isCd = configserverConfig.system().equals(SystemName.cd.value()); // TODO: Want job control or feature flag to control when to run this, for now use a very // long interval to avoid running the maintainer except in CD this.tenantsMaintainerInterval = isCd diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 6351a93e6e6..94cd30de28b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -12,6 +13,7 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.GlobalComponentRegistry; @@ -24,8 +26,10 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; +import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; @@ -53,6 +57,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final Curator curator; private final DeployLogger logger; private final FlagSource flagSource; + private final SecretStore secretStore; public ActivatedModelsBuilder(TenantName tenant, long appGeneration, @@ -71,6 +76,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { this.curator = globalComponentRegistry.getCurator(); this.logger = new SilentDeployLogger(); this.flagSource = globalComponentRegistry.getFlagSource(); + this.secretStore = globalComponentRegistry.getSecretStore(); } @Override @@ -127,9 +133,11 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { configserverConfig.hostedVespa(), zone(), new Rotations(curator, TenantRepository.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId), + ImmutableSet.copyOf(new ContainerEndpointsCache(TenantRepository.getTenantPath(tenant), curator).read(applicationId)), false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating - flagSource); + flagSource, + new TlsSecretsKeys(curator, TenantRepository.getTenantPath(tenant), secretStore).readTlsSecretsKeyFromZookeeper(applicationId)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 07c06f22497..34dcefe05bd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.component.Version; +import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; import com.yahoo.log.LogLevel; @@ -111,7 +112,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { break; buildLatestModelForThisMajor = false; // We have successfully built latest model version, do it only for this major } - catch (OutOfCapacityException | ApplicationLockException e) { + catch (OutOfCapacityException | ApplicationLockException | TransientException e) { // Don't wrap this exception, and don't try to load other model versions as this is (most likely) // caused by the state of the system, not the model version/application combination throw e; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index bbdef71129a..67aa49ea0b9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -164,7 +164,7 @@ class GetConfigProcessor implements Runnable { } private void returnEmpty(JRTServerConfigRequest request) { - log.log(LogLevel.INFO, "Returning empty sentinel config for request from " + request.getClientHostName()); + log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RequestHandlerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RequestHandlerProvider.java new file mode 100644 index 00000000000..6af9034ae96 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RequestHandlerProvider.java @@ -0,0 +1,19 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.tenant.TenantRepository; + +import java.util.Optional; + +/** + * A provider of {@link RequestHandler} instances. A very simplified interface of {@link TenantRepository}. + * + * @author bjorncs + */ +public interface RequestHandlerProvider { + + Optional<RequestHandler> getRequestHandler(TenantName tenantName); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java new file mode 100644 index 00000000000..37c91cf255d --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcRequestHandlerProvider.java @@ -0,0 +1,35 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc; + +import com.google.inject.Inject; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.rpc.security.DefaultRpcAuthorizerProvider; +import com.yahoo.vespa.config.server.tenant.TenantRepository; + +import java.util.Optional; + +/** + * This is a hack to avoid a cyclic dependency involving {@link RpcServer}, {@link DefaultRpcAuthorizerProvider} and {@link TenantRepository}. + * + * @author bjorncs + */ +public class RpcRequestHandlerProvider implements RequestHandlerProvider { + + private volatile RpcServer instance; + + @Inject + public RpcRequestHandlerProvider() {} + + @Override + public Optional<RequestHandler> getRequestHandler(TenantName tenantName) { + if (instance == null) { + throw new IllegalStateException("RpcServer instance has not been registered"); + } + return instance.getRequestHandler(tenantName); + } + + void setInstance(RpcServer instance) { + this.instance = instance; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 1df72522310..e2c1139e4a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -3,12 +3,12 @@ package com.yahoo.vespa.config.server.rpc; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Acceptor; import com.yahoo.jrt.DataValue; import com.yahoo.jrt.Int32Value; @@ -28,16 +28,17 @@ import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import com.yahoo.vespa.config.protocol.Trace; +import com.yahoo.vespa.config.server.GetConfigContext; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.SuperModelRequestHandler; import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.GetConfigContext; import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.MetricUpdaterFactory; +import com.yahoo.vespa.config.server.rpc.security.RpcAuthorizer; import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; import com.yahoo.vespa.config.server.tenant.TenantListener; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -79,8 +80,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { static final int TRACELEVEL_DEBUG = 9; private static final String THREADPOOL_NAME = "rpcserver worker pool"; private static final long SHUTDOWN_TIMEOUT = 60; + private static final int JRT_RPC_TRANSPORT_THREADS = 4; - private final Supervisor supervisor = new Supervisor(new Transport()); + private final Supervisor supervisor = new Supervisor(new Transport(JRT_RPC_TRANSPORT_THREADS)); private Spec spec; private final boolean useRequestVersion; private final boolean hostedVespa; @@ -98,6 +100,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { private final MetricUpdaterFactory metricUpdaterFactory; private final HostLivenessTracker hostLivenessTracker; private final FileServer fileServer; + private final RpcAuthorizer rpcAuthorizer; private final ThreadPoolExecutor executorService; private final FileDownloader downloader; @@ -120,7 +123,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Inject public RpcServer(ConfigserverConfig config, SuperModelRequestHandler superModelRequestHandler, MetricUpdaterFactory metrics, HostRegistries hostRegistries, - HostLivenessTracker hostLivenessTracker, FileServer fileServer) { + HostLivenessTracker hostLivenessTracker, FileServer fileServer, RpcAuthorizer rpcAuthorizer, + RpcRequestHandlerProvider handlerProvider) { this.superModelRequestHandler = superModelRequestHandler; metricUpdaterFactory = metrics; supervisor.setMaxOutputBufferSize(config.maxoutputbuffersize()); @@ -139,44 +143,42 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { this.hostedVespa = config.hostedVespa(); this.canReturnEmptySentinelConfig = config.canReturnEmptySentinelConfig(); this.fileServer = fileServer; + this.rpcAuthorizer = rpcAuthorizer; downloader = fileServer.downloader(); + handlerProvider.setInstance(this); setUpHandlers(); } /** - * Called by reflection from RPC. * Handles RPC method "config.v3.getConfig" requests. * Uses the template pattern to call methods in classes that extend RpcServer. */ - @SuppressWarnings({"UnusedDeclaration"}) - public final void getConfigV3(Request req) { + private void getConfigV3(Request req) { if (log.isLoggable(LogLevel.SPAM)) { log.log(LogLevel.SPAM, getConfigMethodName); } req.detach(); - addToRequestQueue(JRTServerConfigRequestV3.createFromRequest(req)); + rpcAuthorizer.authorizeConfigRequest(req) + .thenRun(() -> addToRequestQueue(JRTServerConfigRequestV3.createFromRequest(req))); } /** - * Called by reflection from RPC. * Returns 0 if server is alive. */ - @SuppressWarnings("UnusedDeclaration") - public final void ping(Request req) { + private void ping(Request req) { req.returnValues().add(new Int32Value(0)); } /** - * Called by reflection from RPC. * Returns a String with statistics data for the server. * * @param req a Request */ - @SuppressWarnings("UnusedDeclaration") - public void printStatistics(Request req) { + private void printStatistics(Request req) { req.returnValues().add(new StringValue("Delayed responses queue size: " + delayedConfigResponses.size())); } + @Override public void run() { log.log(LogLevel.INFO, "Rpc will listen on port " + spec.port()); try { @@ -211,16 +213,15 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { */ private void setUpHandlers() { // The getConfig method in this class will handle RPC calls for getting config - getSupervisor().addMethod(JRTMethods.createConfigV3GetConfigMethod(this, getConfigMethodName)); - getSupervisor().addMethod(new Method("ping", "", "i", this, "ping") + getSupervisor().addMethod(JRTMethods.createConfigV3GetConfigMethod(this::getConfigV3)); + getSupervisor().addMethod(new Method("ping", "", "i", this::ping) .methodDesc("ping") .returnDesc(0, "ret code", "return code, 0 is OK")); - getSupervisor().addMethod(new Method("printStatistics", "", "s", this, "printStatistics") + getSupervisor().addMethod(new Method("printStatistics", "", "s", this::printStatistics) .methodDesc("printStatistics") .returnDesc(0, "statistics", "Statistics for server")); - getSupervisor().addMethod(new Method("filedistribution.serveFile", "si", "is", this, "serveFile")); - getSupervisor().addMethod(new Method("filedistribution.setFileReferencesToDownload", "S", "i", - this, "setFileReferencesToDownload") + getSupervisor().addMethod(new Method("filedistribution.serveFile", "si", "is", this::serveFile)); + getSupervisor().addMethod(new Method("filedistribution.setFileReferencesToDownload", "S", "i", this::setFileReferencesToDownload) .methodDesc("set which file references to download") .paramDesc(0, "file references", "file reference to download") .returnDesc(0, "ret", "0 if success, 1 otherwise")); @@ -389,14 +390,15 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { return GetConfigContext.create(ApplicationId.global(), superModelRequestHandler, trace); } TenantName tenant = optionalTenant.orElse(TenantName.defaultName()); // perhaps needed for non-hosted? - if ( ! hasRequestHandler(tenant)) { + Optional<RequestHandler> requestHandler = getRequestHandler(tenant); + if (requestHandler.isEmpty()) { String msg = TenantRepository.logPre(tenant) + "Unable to find request handler for tenant. Requested from host '" + request.getClientHostName() + "'"; metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); log.log(LogLevel.WARNING, msg); return null; } - RequestHandler handler = getRequestHandler(tenant); + RequestHandler handler = requestHandler.get(); ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName()); if (trace.shouldTrace(TRACELEVEL_DEBUG)) { trace.trace(TRACELEVEL_DEBUG, "Host '" + request.getClientHostName() + "' should have config from application '" + applicationId + "'"); @@ -404,15 +406,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { return GetConfigContext.create(applicationId, handler, trace); } - private boolean hasRequestHandler(TenantName tenant) { - return tenantProviders.containsKey(tenant); - } - - private RequestHandler getRequestHandler(TenantName tenant) { - if (!tenantProviders.containsKey(tenant)) { - throw new IllegalStateException("No request handler for " + tenant); - } - return tenantProviders.get(tenant).getRequestHandler(); + Optional<RequestHandler> getRequestHandler(TenantName tenant) { + return Optional.ofNullable(tenantProviders.get(tenant)) + .map(TenantHandlerProvider::getRequestHandler); } void delayResponse(JRTServerConfigRequest request, GetConfigContext context) { @@ -547,21 +543,26 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } - @SuppressWarnings("UnusedDeclaration") - public final void serveFile(Request request) { + private void serveFile(Request request) { request.detach(); - FileServer.Receiver receiver = new ChunkedFileReceiver(request.target()); - fileServer.serveFile(request.parameters().get(0).asString(), request.parameters().get(1).asInt32() == 0, request, receiver); + rpcAuthorizer.authorizeFileRequest(request) + .thenRun(() -> { // okay to do in authorizer thread as serveFile is async + FileServer.Receiver receiver = new ChunkedFileReceiver(request.target()); + fileServer.serveFile(request.parameters().get(0).asString(), request.parameters().get(1).asInt32() == 0, request, receiver); + }); } - @SuppressWarnings({"UnusedDeclaration"}) - public final void setFileReferencesToDownload(Request req) { - String[] fileReferenceStrings = req.parameters().get(0).asStringArray(); - Stream.of(fileReferenceStrings) - .map(FileReference::new) - .forEach(fileReference -> downloader.downloadIfNeeded( - new FileReferenceDownload(fileReference, false /* downloadFromOtherSourceIfNotFound */))); - req.returnValues().add(new Int32Value(0)); + private void setFileReferencesToDownload(Request req) { + req.detach(); + rpcAuthorizer.authorizeFileRequest(req) + .thenRun(() -> { // okay to do in authorizer thread as downloadIfNeeded is async + String[] fileReferenceStrings = req.parameters().get(0).asStringArray(); + Stream.of(fileReferenceStrings) + .map(FileReference::new) + .forEach(fileReference -> downloader.downloadIfNeeded( + new FileReferenceDownload(fileReference, false /* downloadFromOtherSourceIfNotFound */))); + req.returnValues().add(new Int32Value(0)); + }); } HostLivenessTracker hostLivenessTracker() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java new file mode 100644 index 00000000000..20435d96068 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/AuthorizationException.java @@ -0,0 +1,17 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +/** + * @author bjorncs + */ +class AuthorizationException extends RuntimeException { + + AuthorizationException(String message) { + super(message); + } + + AuthorizationException(String message, Throwable cause) { + super(message, cause); + } +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java new file mode 100644 index 00000000000..c7bbecc157c --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java @@ -0,0 +1,57 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.security.NodeIdentifier; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.security.tls.TransportSecurityUtils; +import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.rpc.RequestHandlerProvider; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; + +/** + * A provider for {@link RpcAuthorizer}. The instance provided is dependent on the configuration of the configserver. + * + * @author bjorncs + */ +public class DefaultRpcAuthorizerProvider implements Provider<RpcAuthorizer> { + + private final RpcAuthorizer rpcAuthorizer; + + @Inject + public DefaultRpcAuthorizerProvider(ConfigserverConfig config, + NodeIdentifier nodeIdentifier, + HostRegistries hostRegistries, + RequestHandlerProvider handlerProvider, + FlagSource flagSource) { + String authorizerMode = Flags.CONFIGSERVER_RPC_AUTHORIZER.bindTo(flagSource).value(); + boolean useMultiTenantAuthorizer = + TransportSecurityUtils.isTransportSecurityEnabled() && config.multitenant() && config.hostedVespa() && !authorizerMode.equals("disable"); + this.rpcAuthorizer = + useMultiTenantAuthorizer + ? new MultiTenantRpcAuthorizer(nodeIdentifier, hostRegistries, handlerProvider, toMultiTenantRpcAuthorizerMode(authorizerMode), getThreadPoolSize(config)) + : new NoopRpcAuthorizer(); + } + + private static MultiTenantRpcAuthorizer.Mode toMultiTenantRpcAuthorizerMode(String authorizerMode) { + switch (authorizerMode) { + case "log-only": return MultiTenantRpcAuthorizer.Mode.LOG_ONLY; + case "enforce": return MultiTenantRpcAuthorizer.Mode.ENFORCE; + default: throw new IllegalArgumentException("Invalid authorizer mode: " + authorizerMode); + } + } + + private static int getThreadPoolSize(ConfigserverConfig config) { + return config.numRpcThreads() != 0 ? config.numRpcThreads() : 8; + } + + @Override + public RpcAuthorizer get() { + return rpcAuthorizer; + } + + @Override + public void deconstruct() {} +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DummyNodeIdentifierProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DummyNodeIdentifierProvider.java new file mode 100644 index 00000000000..8666a47f09f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DummyNodeIdentifierProvider.java @@ -0,0 +1,36 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +import com.google.inject.Inject; +import com.yahoo.config.provision.security.NodeIdentifier; +import com.yahoo.config.provision.security.NodeIdentity; +import com.yahoo.container.di.componentgraph.Provider; + +import java.security.cert.X509Certificate; +import java.util.List; + +/** + * @author bjorncs + */ +public class DummyNodeIdentifierProvider implements Provider<NodeIdentifier> { + + private final ThrowingNodeIdentifier instance = new ThrowingNodeIdentifier(); + + @Inject + public DummyNodeIdentifierProvider() {} + + @Override + public NodeIdentifier get() { + return instance; + } + + @Override + public void deconstruct() {} + + private static class ThrowingNodeIdentifier implements NodeIdentifier { + @Override + public NodeIdentity identifyNode(List<X509Certificate> peerCertificateChain) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/GlobalConfigAuthorizationPolicy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/GlobalConfigAuthorizationPolicy.java new file mode 100644 index 00000000000..cc1125b6cc6 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/GlobalConfigAuthorizationPolicy.java @@ -0,0 +1,52 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.RoutingConfig; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.config.ConfigKey; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; + +/** + * Specifies which node type that are allowed to access global configuration + * + * @author bjorncs + */ +enum GlobalConfigAuthorizationPolicy { + + LB_SERVICES(new LbServicesConfig.Builder(), NodeType.proxy), + ROUTING(new RoutingConfig.Builder(), NodeType.tenant); // TODO Remove handling of RoutingConfig when YCA filter is removed + + final String namespace; + final String name; + final EnumSet<NodeType> allowedToAccess; + + GlobalConfigAuthorizationPolicy(ConfigInstance.Builder builder, NodeType... allowedToAccess) { + this.namespace = builder.getDefNamespace(); + this.name = builder.getDefName(); + this.allowedToAccess = EnumSet.copyOf(List.of(allowedToAccess)); + } + + static void verifyAccessAllowed(ConfigKey<?> configKey, NodeType nodeType) { + GlobalConfigAuthorizationPolicy policy = findPolicyFromConfigKey(configKey); + if (!policy.allowedToAccess.contains(nodeType)) { + String message = String.format( + "Node with type '%s' is not allowed to access global config [%s]", + nodeType, configKey); + throw new AuthorizationException(message); + } + } + + private static GlobalConfigAuthorizationPolicy findPolicyFromConfigKey(ConfigKey<?> configKey) { + return Arrays.stream(values()) + .filter(policy -> policy.namespace.equals(configKey.getNamespace()) && policy.name.equals(configKey.getName())) + .findAny() + .orElseThrow(() -> new AuthorizationException(String.format("No policy defined for global config [%s]", configKey))); + } + +} + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java new file mode 100644 index 00000000000..15e52e48c3a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -0,0 +1,227 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.FileReference; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.security.NodeIdentifier; +import com.yahoo.config.provision.security.NodeIdentifierException; +import com.yahoo.config.provision.security.NodeIdentity; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.SecurityContext; +import com.yahoo.log.LogLevel; +import com.yahoo.security.tls.MixedMode; +import com.yahoo.security.tls.TransportSecurityUtils; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.rpc.RequestHandlerProvider; + +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.function.BiConsumer; +import java.util.logging.Logger; + + +/** + * A {@link RpcAuthorizer} that perform access control for configserver RPC methods when TLS and multi-tenant mode are enabled. + * + * @author bjorncs + */ +public class MultiTenantRpcAuthorizer implements RpcAuthorizer { + + public enum Mode { LOG_ONLY, ENFORCE } + + private static final Logger log = Logger.getLogger(MultiTenantRpcAuthorizer.class.getName()); + + private final NodeIdentifier nodeIdentifier; + private final HostRegistry<TenantName> hostRegistry; + private final RequestHandlerProvider handlerProvider; + private final Executor executor; + private final Mode mode; + + public MultiTenantRpcAuthorizer(NodeIdentifier nodeIdentifier, + HostRegistries hostRegistries, + RequestHandlerProvider handlerProvider, + Mode mode, + int threadPoolSize) { + this(nodeIdentifier, + hostRegistries.getTenantHostRegistry(), + handlerProvider, + Executors.newFixedThreadPool(threadPoolSize, new DaemonThreadFactory("multi-tenant-rpc-authorizer-")), + mode); + } + + MultiTenantRpcAuthorizer(NodeIdentifier nodeIdentifier, + HostRegistry<TenantName> hostRegistry, + RequestHandlerProvider handlerProvider, + Executor executor, + Mode mode) { + this.nodeIdentifier = nodeIdentifier; + this.hostRegistry = hostRegistry; + this.handlerProvider = handlerProvider; + this.executor = executor; + this.mode = mode; + } + + @Override + public CompletableFuture<Void> authorizeConfigRequest(Request request) { + return doAsyncAuthorization(request, this::doConfigRequestAuthorization); + } + + @Override + public CompletableFuture<Void> authorizeFileRequest(Request request) { + return doAsyncAuthorization(request, this::doFileRequestAuthorization); + } + + private CompletableFuture<Void> doAsyncAuthorization(Request request, BiConsumer<Request, NodeIdentity> authorizer) { + return CompletableFuture.runAsync( + () -> { + try { + getPeerIdentity(request) + .ifPresent(peerIdentity -> authorizer.accept(request, peerIdentity)); + log.log(LogLevel.DEBUG, () -> String.format("Authorization succeeded for request '%s' from '%s'", + request.methodName(), request.target().toString())); + } catch (Throwable t) { + handleAuthorizationFailure(request, t); + } + }, + executor); + } + + private void doConfigRequestAuthorization(Request request, NodeIdentity peerIdentity) { + switch (peerIdentity.nodeType()) { + case config: + return; // configserver is allowed to access all config + case proxy: + case tenant: + case host: + JRTServerConfigRequestV3 configRequest = JRTServerConfigRequestV3.createFromRequest(request); + ConfigKey<?> configKey = configRequest.getConfigKey(); + if (isConfigKeyForGlobalConfig(configKey)) { + GlobalConfigAuthorizationPolicy.verifyAccessAllowed(configKey, peerIdentity.nodeType()); + return; // global config access ok + } else { + String hostname = configRequest.getClientHostName(); + Optional<TenantName> tenantName = Optional.ofNullable(hostRegistry.getKeyForHost(hostname)); + if (tenantName.isEmpty()) { + if (isConfigKeyForSentinelConfig(configKey)) { + return; // config processor will return empty sentinel config for unknown nodes + } + throw new AuthorizationException(String.format("Host '%s' not found in host registry for [%s]", hostname, configKey)); + } + RequestHandler tenantHandler = getTenantHandler(tenantName.get()); + ApplicationId resolvedApplication = tenantHandler.resolveApplicationId(hostname); + ApplicationId peerOwner = applicationId(peerIdentity); + if (peerOwner.equals(resolvedApplication)) { + return; // allowed to access + } + throw new AuthorizationException( + String.format( + "Peer is not allowed to access config for owned by %s. Peer is owned by %s", + resolvedApplication.toShortString(), peerOwner.toShortString())); + } + default: + throw new AuthorizationException(String.format("'%s' nodes are not allowed to access config", peerIdentity.nodeType())); + } + } + + private void doFileRequestAuthorization(Request request, NodeIdentity peerIdentity) { + switch (peerIdentity.nodeType()) { + case config: + return; // configserver is allowed to access all files + case proxy: + case tenant: + case host: + ApplicationId peerOwner = applicationId(peerIdentity); + FileReference requestedFile = new FileReference(request.parameters().get(0).asString()); + RequestHandler tenantHandler = getTenantHandler(peerOwner.tenant()); + Set<FileReference> filesOwnedByApplication = tenantHandler.listFileReferences(peerOwner); + if (filesOwnedByApplication.contains(requestedFile)) { + return; // allowed to access + } + throw new AuthorizationException(String.format("Peer is not allowed to access file %s. Peer is owned by %s", requestedFile.value(), peerOwner.toShortString())); + default: + throw new AuthorizationException(String.format("'%s' nodes are not allowed to access files", peerIdentity.nodeType())); + } + } + + private void handleAuthorizationFailure(Request request, Throwable throwable) { + String errorMessage = String.format("For request '%s' from '%s' (mode=%s): %s", request.methodName(), request.target().toString(), mode.toString(), throwable.getMessage()); + log.log(LogLevel.INFO, errorMessage); + log.log(LogLevel.DEBUG, throwable, throwable::getMessage); + if (mode == Mode.ENFORCE) { + JrtErrorCode error = throwable instanceof AuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; + request.setError(error.code, errorMessage); + request.returnRequest(); + throwUnchecked(throwable); // rethrow exception to ensure that subsequent completion stages are not executed (don't execute implementation of rpc method). + } + } + + // TODO Make peer identity mandatory once TLS mixed mode is removed + private Optional<NodeIdentity> getPeerIdentity(Request request) { + Optional<SecurityContext> securityContext = request.target().getSecurityContext(); + if (securityContext.isEmpty()) { + if (TransportSecurityUtils.getInsecureMixedMode() == MixedMode.DISABLED) { + throw new IllegalStateException("Security context missing"); // security context should always be present + } + return Optional.empty(); // client choose to communicate over insecure channel + } + List<X509Certificate> certChain = securityContext.get().peerCertificateChain(); + if (certChain.isEmpty()) { + throw new IllegalStateException("Client authentication is not enforced!"); // clients should be required to authenticate when TLS is enabled + } + try { + NodeIdentity identity = nodeIdentifier.identifyNode(certChain); + log.log(LogLevel.DEBUG, () -> String.format("Client '%s' identified as %s", request.target().toString(), identity.toString())); + return Optional.of(identity); + } catch (NodeIdentifierException e) { + throw new AuthorizationException("Failed to identity peer: " + e.getMessage(), e); + } + } + + private static boolean isConfigKeyForGlobalConfig(ConfigKey<?> configKey) { + return "*".equals(configKey.getConfigId()); + } + + private static boolean isConfigKeyForSentinelConfig(ConfigKey<?> configKey) { + return SentinelConfig.getDefName().equals(configKey.getName()) + && SentinelConfig.getDefNamespace().equals(configKey.getNamespace()); + } + + private static ApplicationId applicationId(NodeIdentity peerIdentity) { + return peerIdentity.applicationId() + .orElseThrow(() -> new AuthorizationException("Peer node is not associated with an application: " + peerIdentity.toString())); + } + + private RequestHandler getTenantHandler(TenantName tenantName) { + return handlerProvider.getRequestHandler(tenantName) + .orElseThrow(() -> new AuthorizationException(String.format("No handler exists for tenant '%s'", tenantName.value()))); + } + + @SuppressWarnings("unchecked") + private static <T extends Throwable> void throwUnchecked(Throwable t) throws T { + throw (T)t; + } + + private enum JrtErrorCode { + UNAUTHORIZED(1), + AUTHORIZATION_FAILED(2); + + final int code; + + JrtErrorCode(int errorOffset) { + this.code = 0x20000 + errorOffset; + } + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/NoopRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/NoopRpcAuthorizer.java new file mode 100644 index 00000000000..5eb35b70d0f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/NoopRpcAuthorizer.java @@ -0,0 +1,24 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +import com.yahoo.jrt.Request; + +import java.util.concurrent.CompletableFuture; + +/** + * A {@link RpcAuthorizer} that allow all RPC requests. + * + * @author bjorncs + */ +public class NoopRpcAuthorizer implements RpcAuthorizer { + + @Override + public CompletableFuture<Void> authorizeConfigRequest(Request request) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture<Void> authorizeFileRequest(Request request) { + return CompletableFuture.completedFuture(null); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/RpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/RpcAuthorizer.java new file mode 100644 index 00000000000..ccda24530e4 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/RpcAuthorizer.java @@ -0,0 +1,19 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.rpc.security; + +import com.yahoo.jrt.Request; + +import java.util.concurrent.CompletableFuture; + +/** + * Authorization logic for configserver's RPC method + * + * @author bjorncs + */ +public interface RpcAuthorizer { + + CompletableFuture<Void> authorizeConfigRequest(Request request); + + CompletableFuture<Void> authorizeFileRequest(Request request); + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 6a0a4a19737..7fd29368ab3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -41,7 +41,6 @@ public class LocalSession extends Session implements Comparable<LocalSession> { private final SessionPreparer sessionPreparer; private final SessionContext sessionContext; private final File serverDB; - private final SuperModelGenerationCounter superModelGenerationCounter; /** * Create a session. This involves loading the application, validating it and distributing it. @@ -56,7 +55,6 @@ public class LocalSession extends Session implements Comparable<LocalSession> { this.applicationRepo = sessionContext.getApplicationRepo(); this.sessionPreparer = sessionPreparer; this.sessionContext = sessionContext; - this.superModelGenerationCounter = sessionContext.getSuperModelGenerationCounter(); } public ConfigChangeActions prepare(DeployLogger logger, @@ -94,7 +92,6 @@ public class LocalSession extends Session implements Comparable<LocalSession> { public Transaction createActivateTransaction() { zooKeeperClient.createActiveWaiter(); - superModelGenerationCounter.increment(); // TODO jvenstad: I hope this counter isn't used for serious things, as it's updated way ahead of activation. Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; @@ -120,8 +117,7 @@ public class LocalSession extends Session implements Comparable<LocalSession> { /** Add transactions to delete this session to the given nested transaction */ public void delete(NestedTransaction transaction) { transaction.add(zooKeeperClient.deleteTransaction(), FileTransaction.class); - transaction.add(FileTransaction.from(FileOperations.delete(serverDB.getAbsolutePath())), SuperModelGenerationCounter.IncrementTransaction.class); - transaction.add(superModelGenerationCounter.incrementTransaction()); + transaction.add(FileTransaction.from(FileOperations.delete(serverDB.getAbsolutePath()))); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index acfd81b33bf..f6d73f33504 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -1,9 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.concurrent.InThreadExecutorService; +import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -12,10 +16,12 @@ import com.yahoo.vespa.curator.Curator; import java.io.File; import java.io.FilenameFilter; import java.time.Clock; +import java.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; @@ -33,23 +39,24 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { private final long sessionLifetime; // in seconds private final Clock clock; private final Curator curator; + private final Executor zkWatcherExecutor; - public LocalSessionRepo(TenantFileSystemDirs tenantFileSystemDirs, LocalSessionLoader loader, - Clock clock, long sessionLifeTime, Curator curator) { - this(clock, curator, sessionLifeTime); + public LocalSessionRepo(TenantName tenantName, GlobalComponentRegistry registry, TenantFileSystemDirs tenantFileSystemDirs, LocalSessionLoader loader) { + this(registry.getClock(), registry.getCurator(), registry.getConfigserverConfig().sessionLifetime(), + command -> registry.getZkWatcherExecutor().execute(tenantName, command)); loadSessions(tenantFileSystemDirs.sessionsPath(), loader); } // Constructor public only for testing public LocalSessionRepo(Clock clock, Curator curator) { - this(clock, curator, TimeUnit.DAYS.toMillis(1)); + this(clock, curator, Duration.ofDays(1).toMillis(), Runnable::run); } - // Constructor public only for testing - private LocalSessionRepo(Clock clock, Curator curator, long sessionLifetime) { + private LocalSessionRepo(Clock clock, Curator curator, long sessionLifetime, Executor zkWatcherExecutor) { this.clock = clock; this.curator = curator; this.sessionLifetime = sessionLifetime; + this.zkWatcherExecutor = zkWatcherExecutor; } @Override @@ -58,7 +65,7 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); long sessionId = session.getSessionId(); Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - sessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this)); + sessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); } private void loadSessions(File applicationsDir, LocalSessionLoader loader) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 37082888d70..53a472c2b67 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.NodeCacheListener; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -21,17 +21,18 @@ import java.util.logging.Logger; public class LocalSessionStateWatcher { private static final Logger log = Logger.getLogger(LocalSessionStateWatcher.class.getName()); - // One thread pool for all instances of this class - private static final Executor executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(LocalSessionStateWatcher.class.getName())); private final Curator.FileCache fileCache; private final LocalSession session; private final LocalSessionRepo localSessionRepo; + private final Executor zkWatcherExecutor; - LocalSessionStateWatcher(Curator.FileCache fileCache, LocalSession session, LocalSessionRepo localSessionRepo) { + LocalSessionStateWatcher(Curator.FileCache fileCache, LocalSession session, + LocalSessionRepo localSessionRepo, Executor zkWatcherExecutor) { this.fileCache = fileCache; this.session = session; this.localSessionRepo = localSessionRepo; + this.zkWatcherExecutor = zkWatcherExecutor; this.fileCache.start(); this.fileCache.addListener(this::nodeChanged); } @@ -60,7 +61,7 @@ public class LocalSessionStateWatcher { } public void nodeChanged() { - executor.execute(() -> { + zkWatcherExecutor.execute(() -> { try { ChildData data = fileCache.getCurrentData(); if (data != null) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index a5e74d3b85f..5bf70c55f9e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -6,12 +6,17 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; import java.time.Clock; import java.time.Duration; import java.util.LinkedHashSet; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -29,6 +34,8 @@ public final class PrepareParams { static final String VERBOSE_PARAM_NAME = "verbose"; static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String ROTATIONS_PARAM_NAME = "rotations"; + static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; + static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -38,9 +45,12 @@ public final class PrepareParams { private final boolean isBootstrap; private final Optional<Version> vespaVersion; private final Set<Rotation> rotations; + private final List<ContainerEndpoint> containerEndpoints; + private final Optional<String> tlsSecretsKeyName; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, - boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, Set<Rotation> rotations) { + boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, Set<Rotation> rotations, + List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -49,6 +59,11 @@ public final class PrepareParams { this.isBootstrap = isBootstrap; this.vespaVersion = vespaVersion; this.rotations = rotations; + this.containerEndpoints = containerEndpoints; + if ((rotations != null && !rotations.isEmpty()) && !containerEndpoints.isEmpty()) { + throw new IllegalArgumentException("Cannot set both rotations and containerEndpoints"); + } + this.tlsSecretsKeyName = tlsSecretsKeyName; } public static class Builder { @@ -61,6 +76,8 @@ public final class PrepareParams { private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(30)); private Optional<Version> vespaVersion = Optional.empty(); private Set<Rotation> rotations; + private List<ContainerEndpoint> containerEndpoints = List.of(); + private Optional<String> tlsSecretsKeyName = Optional.empty(); public Builder() { } @@ -119,22 +136,37 @@ public final class PrepareParams { return this; } + public Builder containerEndpoints(String serialized) { + if (serialized == null) return this; + Slime slime = SlimeUtils.jsonToSlime(serialized); + containerEndpoints = ContainerEndpointSerializer.endpointListFromSlime(slime); + return this; + } + + public Builder tlsSecretsKeyName(String tlsSecretsKeyName) { + this.tlsSecretsKeyName = Optional.ofNullable(tlsSecretsKeyName) + .filter(s -> ! s.isEmpty()); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, rotations); + verbose, isBootstrap, vespaVersion, rotations, containerEndpoints, tlsSecretsKeyName); } } public static PrepareParams fromHttpRequest(HttpRequest request, TenantName tenant, Duration barrierTimeout) { return new Builder().ignoreValidationErrors(request.getBooleanProperty(IGNORE_VALIDATION_PARAM_NAME)) - .dryRun(request.getBooleanProperty(DRY_RUN_PARAM_NAME)) - .verbose(request.getBooleanProperty(VERBOSE_PARAM_NAME)) - .timeoutBudget(SessionHandler.getTimeoutBudget(request, barrierTimeout)) - .applicationId(createApplicationId(request, tenant)) - .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) - .rotations(request.getProperty(ROTATIONS_PARAM_NAME)) - .build(); + .dryRun(request.getBooleanProperty(DRY_RUN_PARAM_NAME)) + .verbose(request.getBooleanProperty(VERBOSE_PARAM_NAME)) + .timeoutBudget(SessionHandler.getTimeoutBudget(request, barrierTimeout)) + .applicationId(createApplicationId(request, tenant)) + .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) + .rotations(request.getProperty(ROTATIONS_PARAM_NAME)) + .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) + .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) + .build(); } private static ApplicationId createApplicationId(HttpRequest request, TenantName tenant) { @@ -164,8 +196,15 @@ public final class PrepareParams { /** Returns the Vespa version the nodes running the prepared system should have, or empty to use the system version */ public Optional<Version> vespaVersion() { return vespaVersion; } + /** Returns the global rotations that should be made available for this deployment */ + // TODO: Remove this once all applications have to switched to containerEndpoints public Set<Rotation> rotations() { return rotations; } + /** Returns the container endpoints that should be made available for this deployment. One per cluster */ + public List<ContainerEndpoint> containerEndpoints() { + return containerEndpoints; + } + public boolean ignoreValidationErrors() { return ignoreValidationErrors; } @@ -184,4 +223,7 @@ public final class PrepareParams { return timeoutBudget; } + public Optional<String> tlsSecretsKeyName() { + return tlsSecretsKeyName; + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index bbb06515bef..e0727effeda 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -3,18 +3,24 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; +import com.yahoo.concurrent.InThreadExecutorService; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; -import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -26,6 +32,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; @@ -33,10 +40,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper, set for example - * by the prepare HTTP handler on another configserver. The zookeeper state watched in this class is shared - * between all config servers, so it should not modify any global state, because the operation will be performed - * on all servers. The repo can be regarded as read only from the POV of the configserver. + * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper. The zookeeper state watched in + * this class is shared between all config servers, so it should not modify any global state, because the operation + * will be performed on all servers. The repo can be regarded as read only from the POV of the configserver. * * @author Vegard Havdal * @author Ulf Lilleengen @@ -44,9 +50,6 @@ import java.util.stream.Collectors; public class RemoteSessionRepo extends SessionRepo<RemoteSession> { private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); - // One thread pool for all instances of this class - private static final ExecutorService pathChildrenExecutor = - Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(RemoteSessionRepo.class.getName())); private final Curator curator; private final Path sessionsPath; @@ -55,31 +58,29 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { private final ReloadHandler reloadHandler; private final TenantName tenantName; private final MetricUpdater metrics; + private final FlagSource flagSource; private final Curator.DirectoryCache directoryCache; private final TenantApplications applicationRepo; + private final Executor zkWatcherExecutor; - /** - * @param curator a {@link Curator} instance. - * @param remoteSessionFactory a {@link com.yahoo.vespa.config.server.session.RemoteSessionFactory} - * @param reloadHandler a {@link com.yahoo.vespa.config.server.ReloadHandler} - * @param tenantName a {@link TenantName} instance. - * @param applicationRepo a {@link TenantApplications} instance. - */ - public RemoteSessionRepo(Curator curator, + public RemoteSessionRepo(GlobalComponentRegistry registry, RemoteSessionFactory remoteSessionFactory, ReloadHandler reloadHandler, TenantName tenantName, - TenantApplications applicationRepo, - MetricUpdater metricUpdater) { - this.curator = curator; + TenantApplications applicationRepo) { + + this.curator = registry.getCurator(); this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.applicationRepo = applicationRepo; this.remoteSessionFactory = remoteSessionFactory; this.reloadHandler = reloadHandler; this.tenantName = tenantName; - this.metrics = metricUpdater; + this.metrics = registry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + this.flagSource = registry.getFlagSource(); + StripedExecutor<TenantName> zkWatcherExecutor = registry.getZkWatcherExecutor(); + this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); initializeSessions(); - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, pathChildrenExecutor); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, registry.getZkCacheExecutor()); this.directoryCache.addListener(this::childEvent); this.directoryCache.start(); } @@ -94,6 +95,8 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { this.metrics = null; this.directoryCache = null; this.applicationRepo = null; + this.flagSource = new InMemoryFlagSource(); + this.zkWatcherExecutor = Runnable::run; } public List<Long> getSessions() { @@ -104,6 +107,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { int deleted = 0; for (long sessionId : getSessions()) { RemoteSession session = getSession(sessionId); + if (session == null) continue; // Internal sessions not in synch with zk, continue Instant created = Instant.ofEpochSecond(session.getCreateTime()); if (sessionHasExpired(created, expiryTime)) { log.log(LogLevel.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); @@ -128,7 +132,6 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { return children.stream().map(Long::parseLong).collect(Collectors.toList()); } - // TODO: Add sessions in parallel private void initializeSessions() throws NumberFormatException { getSessions().forEach(this::sessionAdded); } @@ -158,16 +161,17 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { */ private void sessionAdded(long sessionId) { try { - log.log(LogLevel.DEBUG, "Adding session to RemoteSessionRepo: " + sessionId); + log.log(LogLevel.DEBUG, () -> "Adding session to RemoteSessionRepo: " + sessionId); RemoteSession session = remoteSessionFactory.createSession(sessionId); Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); fileCache.addListener(this::nodeChanged); loadSessionIfActive(session); - sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics)); + sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); addSession(session); metrics.incAddedSessions(); } catch (Exception e) { + if (Flags.CONFIG_SERVER_FAIL_IF_ACTIVE_SESSION_CANNOT_BE_LOADED.bindTo(flagSource).value()) throw e; log.log(Level.WARNING, "Failed loading session " + sessionId + ": No config for this session can be served", e); } } @@ -181,15 +185,11 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { private void loadSessionIfActive(RemoteSession session) { for (ApplicationId applicationId : applicationRepo.activeApplications()) { - try { - if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { - log.log(LogLevel.DEBUG, "Found active application for session " + session.getSessionId() + " , loading it"); - reloadHandler.reloadConfig(session.ensureApplicationLoaded()); - log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId); - return; - } - } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + "Skipping loading of application '" + applicationId + "': " + Exceptions.toMessageString(e)); + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { + log.log(LogLevel.DEBUG, () -> "Found active application for session " + session.getSessionId() + " , loading it"); + reloadHandler.reloadConfig(session.ensureApplicationLoaded()); + log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); + return; } } } @@ -207,39 +207,42 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { } private void nodeChanged() { - Multiset<Session.Status> sessionMetrics = HashMultiset.create(); - for (RemoteSession session : listSessions()) { - sessionMetrics.add(session.getStatus()); - } - metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); - metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); - metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); - metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); - } - - private void childEvent(CuratorFramework framework, PathChildrenCacheEvent event) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got child event: " + event); - } - switch (event.getType()) { - case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; - case CHILD_REMOVED: - sessionsChanged(); - break; - case CONNECTION_RECONNECTED: - sessionsChanged(); - break; - } + zkWatcherExecutor.execute(() -> { + Multiset<Session.Status> sessionMetrics = HashMultiset.create(); + for (RemoteSession session : listSessions()) { + sessionMetrics.add(session.getStatus()); + } + metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); + metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); + metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); + metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); + }); + } + + @SuppressWarnings("unused") + private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { + zkWatcherExecutor.execute(() -> { + log.log(LogLevel.DEBUG, () -> "Got child event: " + event); + switch (event.getType()) { + case CHILD_ADDED: + sessionsChanged(); + synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); + break; + case CHILD_REMOVED: + sessionsChanged(); + break; + case CONNECTION_RECONNECTED: + sessionsChanged(); + break; + } + }); } private void synchronizeOnNew(List<Long> sessionList) { for (long sessionId : sessionList) { RemoteSession session = getSession(sessionId); if (session == null) continue; // session might have been deleted after getting session list - log.log(LogLevel.DEBUG, session.logPre() + "Confirming upload for session " + sessionId); + log.log(LogLevel.DEBUG, () -> session.logPre() + "Confirming upload for session " + sessionId); session.confirmUpload(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java index ef59e28f458..653a2616cbe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java @@ -1,17 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.NodeCacheListener; import java.util.concurrent.Executor; -import java.util.concurrent.Executors; import java.util.logging.Logger; /** @@ -24,24 +23,25 @@ public class RemoteSessionStateWatcher { private static final Logger log = Logger.getLogger(RemoteSessionStateWatcher.class.getName()); // One thread pool for all instances of this class - private static final Executor executor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(RemoteSessionStateWatcher.class.getName())); private final Curator.FileCache fileCache; private final ReloadHandler reloadHandler; private final RemoteSession session; private final MetricUpdater metrics; - + private final Executor zkWatcherExecutor; RemoteSessionStateWatcher(Curator.FileCache fileCache, ReloadHandler reloadHandler, RemoteSession session, - MetricUpdater metrics) { + MetricUpdater metrics, + Executor zkWatcherExecutor) { this.fileCache = fileCache; this.reloadHandler = reloadHandler; this.session = session; this.metrics = metrics; this.fileCache.start(); this.fileCache.addListener(this::nodeChanged); + this.zkWatcherExecutor = zkWatcherExecutor; } private void sessionChanged(Session.Status status) { @@ -73,7 +73,7 @@ public class RemoteSessionStateWatcher { } public void nodeChanged() { - executor.execute(() -> { + zkWatcherExecutor.execute(() -> { try { ChildData data = fileCache.getCurrentData(); if (data != null) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java index f212e1a1486..0495a51514c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java @@ -22,19 +22,17 @@ public class SessionContext { private final File serverDBSessionDir; private final TenantApplications applicationRepo; private final HostValidator<ApplicationId> hostRegistry; - private final SuperModelGenerationCounter superModelGenerationCounter; private final FlagSource flagSource; public SessionContext(ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient, File serverDBSessionDir, TenantApplications applicationRepo, - HostValidator<ApplicationId> hostRegistry, SuperModelGenerationCounter superModelGenerationCounter, + HostValidator<ApplicationId> hostRegistry, FlagSource flagSource) { this.applicationPackage = applicationPackage; this.sessionZooKeeperClient = sessionZooKeeperClient; this.serverDBSessionDir = serverDBSessionDir; this.applicationRepo = applicationRepo; this.hostRegistry = hostRegistry; - this.superModelGenerationCounter = superModelGenerationCounter; this.flagSource = flagSource; } @@ -56,10 +54,6 @@ public class SessionContext { public HostValidator<ApplicationId> getHostValidator() { return hostRegistry; } - public SuperModelGenerationCounter getSuperModelGenerationCounter() { - return superModelGenerationCounter; - } - public FlagSource getFlagSource() { return flagSource; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index d8ba5890545..cc46a157b34 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -46,7 +46,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { private final Path sessionsPath; private final TenantFileSystemDirs tenantFileSystemDirs; private final HostValidator<ApplicationId> hostRegistry; - private final SuperModelGenerationCounter superModelGenerationCounter; private final TenantName tenant; private final String serverId; private final Optional<NodeFlavors> nodeFlavors; @@ -67,7 +66,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { this.sessionsPath = TenantRepository.getSessionsPath(tenant); this.applicationRepo = applicationRepo; this.tenantFileSystemDirs = tenantFileSystemDirs; - this.superModelGenerationCounter = globalComponentRegistry.getSuperModelGenerationCounter(); this.serverId = globalComponentRegistry.getConfigserverConfig().serverId(); this.nodeFlavors = globalComponentRegistry.getZone().nodeFlavors(); this.clock = globalComponentRegistry.getClock(); @@ -113,7 +111,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Creating upload waiter for session " + sessionId); Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Done creating upload waiter for session " + sessionId); - SessionContext context = new SessionContext(applicationPackage, sessionZKClient, getSessionAppDir(sessionId), applicationRepo, hostRegistry, superModelGenerationCounter, flagSource); + SessionContext context = new SessionContext(applicationPackage, sessionZKClient, getSessionAppDir(sessionId), applicationRepo, hostRegistry, flagSource); LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, context); log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); waiter.awaitCompletion(timeoutBudget.timeLeft()); @@ -182,8 +180,8 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { sessionIdPath, serverId, nodeFlavors); - SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, - hostRegistry, superModelGenerationCounter, flagSource); + SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, + hostRegistry, flagSource); return new LocalSession(tenant, sessionId, sessionPreparer, context); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index f0ceeb186fe..54c96c0461d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -2,21 +2,24 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.lang.SettableOptional; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -30,7 +33,10 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.Rotations; +import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import org.xml.sax.SAXException; @@ -40,6 +46,7 @@ import javax.xml.transform.TransformerException; import java.io.IOException; import java.net.URI; import java.time.Instant; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,6 +72,7 @@ public class SessionPreparer { private final Curator curator; private final Zone zone; private final FlagSource flagSource; + private final SecretStore secretStore; @Inject public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry, @@ -75,7 +83,8 @@ public class SessionPreparer { ConfigDefinitionRepo configDefinitionRepo, Curator curator, Zone zone, - FlagSource flagSource) { + FlagSource flagSource, + SecretStore secretStore) { this.modelFactoryRegistry = modelFactoryRegistry; this.fileDistributionFactory = fileDistributionFactory; this.hostProvisionerProvider = hostProvisionerProvider; @@ -85,6 +94,7 @@ public class SessionPreparer { this.curator = curator; this.zone = zone; this.flagSource = flagSource; + this.secretStore = secretStore; } /** @@ -108,6 +118,11 @@ public class SessionPreparer { if ( ! params.isDryRun()) { preparation.writeStateZK(); preparation.writeRotZK(); + preparation.writeTlsZK(); + var globalServiceId = context.getApplicationPackage().getDeployment() + .map(DeploymentSpec::fromXml) + .flatMap(DeploymentSpec::globalServiceId); + preparation.writeContainerEndpointsZK(globalServiceId); preparation.distribute(); } log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + @@ -132,9 +147,13 @@ public class SessionPreparer { /** The version of Vespa the application to be prepared specifies for its nodes */ final com.yahoo.component.Version vespaVersion; - final Rotations rotations; + final Rotations rotations; // TODO: Remove this once we have migrated fully to container endpoints + final ContainerEndpointsCache containerEndpoints; final Set<Rotation> rotationsSet; + final Set<ContainerEndpoint> endpointsSet; final ModelContext.Properties properties; + private final TlsSecretsKeys tlsSecretsKeys; + private final Optional<TlsSecrets> tlsSecrets; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -153,7 +172,12 @@ public class SessionPreparer { this.applicationId = params.getApplicationId(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.rotations = new Rotations(curator, tenantPath); + this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); this.rotationsSet = getRotations(params.rotations()); + this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore); + this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId); + this.endpointsSet = getEndpoints(params.containerEndpoints()); + this.properties = new ModelContextImpl.Properties(params.getApplicationId(), configserverConfig.multitenant(), ConfigServerSpec.fromConfig(configserverConfig), @@ -163,9 +187,11 @@ public class SessionPreparer { configserverConfig.hostedVespa(), zone, rotationsSet, + endpointsSet, params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), - context.getFlagSource()); + context.getFlagSource(), + tlsSecrets); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -225,6 +251,26 @@ public class SessionPreparer { checkTimeout("write rotations to zookeeper"); } + void writeTlsZK() { + tlsSecretsKeys.writeTlsSecretsKeyToZooKeeper(applicationId, params.tlsSecretsKeyName().orElse(null)); + checkTimeout("write tlsSecretsKey to zookeeper"); + } + + void writeContainerEndpointsZK(Optional<String> globalServiceId) { + if (!params.containerEndpoints().isEmpty()) { // Use endpoints from parameter when explicitly given + containerEndpoints.write(applicationId, params.containerEndpoints()); + } else { // Fall back to writing rotations as container endpoints + if (!rotationsSet.isEmpty()) { + if (globalServiceId.isEmpty()) { + log.log(LogLevel.WARNING, "Want to write rotations " + rotationsSet + " as container endpoints, but " + applicationId + " has no global-service-id. This should not happen"); + return; + } + containerEndpoints.write(applicationId, toContainerEndpoints(globalServiceId.get(), rotationsSet)); + } + } + checkTimeout("write container endpoints to zookeeper"); + } + void distribute() { prepareResult.asList().forEach(modelResult -> modelResult.model .distributeFiles(modelResult.fileDistributionProvider.getFileDistribution())); @@ -242,6 +288,20 @@ public class SessionPreparer { return rotations; } + private Set<ContainerEndpoint> getEndpoints(List<ContainerEndpoint> endpoints) { + if (endpoints == null || endpoints.isEmpty()) { + endpoints = this.containerEndpoints.read(applicationId); + } + return ImmutableSet.copyOf(endpoints); + } + + } + + private static List<ContainerEndpoint> toContainerEndpoints(String globalServceId, Set<Rotation> rotations) { + return List.of(new ContainerEndpoint(globalServceId, + rotations.stream() + .map(Rotation::getId) + .collect(Collectors.toUnmodifiableList()))); } private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, @@ -294,10 +354,9 @@ public class SessionPreparer { * (once for the upgrading case and once for a potential restart action). */ public ConfigChangeActions getConfigChangeActions() { - return new ConfigChangeActions(results.stream(). - map(result -> result.actions). - flatMap(actions -> actions.stream()). - collect(Collectors.toList())); + return new ConfigChangeActions(results.stream().map(result -> result.actions) + .flatMap(actions -> actions.stream()) + .collect(Collectors.toList())); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java new file mode 100644 index 00000000000..4ffce8a697e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -0,0 +1,87 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +import java.util.ArrayList; +import java.util.List; + +/** + * Contains all methods for de-/serializing ContainerEndpoints to/from JSON. + * Also supports de-/serializing lists of these values. + * + * @author ogronnesby + */ +public class ContainerEndpointSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String clusterIdField = "clusterId"; + private static final String namesField = "names"; + + private ContainerEndpointSerializer() {} + + public static ContainerEndpoint endpointFromSlime(Inspector inspector) { + final var clusterId = inspector.field(clusterIdField).asString(); + final var namesInspector = inspector.field(namesField); + + if (clusterId.isEmpty()) { + throw new IllegalStateException("'clusterId' missing on serialized ContainerEndpoint"); + } + + if (! namesInspector.valid()) { + throw new IllegalStateException("'names' missing on serialized ContainerEndpoint"); + } + + final var names = new ArrayList<String>(); + + namesInspector.traverse((ArrayTraverser) (idx, nameInspector) -> { + final var containerName = nameInspector.asString(); + names.add(containerName); + }); + + return new ContainerEndpoint(clusterId, names); + } + + public static List<ContainerEndpoint> endpointListFromSlime(Slime slime) { + final var inspector = slime.get(); + final var endpoints = new ArrayList<ContainerEndpoint>(); + + inspector.traverse((ArrayTraverser) (idx, endpointInspector) -> { + final var containerEndpoint = endpointFromSlime(endpointInspector); + endpoints.add(containerEndpoint); + }); + + return endpoints; + } + + + public static void endpointToSlime(Cursor cursor, ContainerEndpoint endpoint) { + cursor.setString(clusterIdField, endpoint.clusterId().toString()); + + final var namesInspector = cursor.setArray(namesField); + endpoint.names().forEach(namesInspector::addString); + } + + public static Slime endpointListToSlime(List<ContainerEndpoint> endpoints) { + final var slime = new Slime(); + final var cursor = slime.setArray(); + + endpoints.forEach(endpoint -> { + final var endpointCursor = cursor.addObject(); + endpointToSlime(endpointCursor, endpoint); + }); + + return slime; + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java new file mode 100644 index 00000000000..9bce1224d96 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -0,0 +1,62 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Curator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; + + +/** + * Persists assignment of rotations to an application to ZooKeeper. + * The entries are {@link ContainerEndpoint} instances, which keep track of the container + * cluster that is the target, the endpoint name, and the rotation used to + * give availability to that cluster. + * + * This is v2 of that storage in a new directory. Previously we only stored + * the name of the rotation, since all the other information could be + * calculated runtime. + * + * @author ogronnesby + */ +public class ContainerEndpointsCache { + + private final Path cachePath; + private final Curator curator; + + public ContainerEndpointsCache(Path tenantPath, Curator curator) { + this.cachePath = tenantPath.append("containerEndpointsCache/"); + this.curator = curator; + } + + public List<ContainerEndpoint> read(ApplicationId applicationId) { + final var optionalData = curator.getData(applicationPath(applicationId)); + return optionalData + .map(SlimeUtils::jsonToSlime) + .map(ContainerEndpointSerializer::endpointListFromSlime) + .orElseGet(List::of); + } + + public void write(ApplicationId applicationId, List<ContainerEndpoint> endpoints) { + if (endpoints.isEmpty()) return; + + final var slime = ContainerEndpointSerializer.endpointListToSlime(endpoints); + + try { + final var bytes = SlimeUtils.toJsonBytes(slime); + curator.set(applicationPath(applicationId), bytes); + } catch (IOException e) { + throw new UncheckedIOException("Error writing endpoints of: " + applicationId, e); + } + } + + private Path applicationPath(ApplicationId applicationId) { + return cachePath.append(applicationId.serializedForm()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index 88e71d7ddd1..0aed3977625 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -142,15 +142,20 @@ public class Tenant implements TenantHandlerProvider { } /** - * Closes any watchers, thread pools that may react to changes in tenant state, and removes any state - * in filesystem and zookeeper + * Closes any watchers, thread pools that may react to changes in tenant state, + * and removes any session data in filesystem and zookeeper. + * Called by watchers as a reaction to {@link #delete()}. */ - public void close() { - tenantFileSystemDirs.delete(); - remoteSessionRepo.close(); - applicationRepo.close(); - localSessionRepo.deleteAllSessions(); - curator.delete(path); + void close() { + tenantFileSystemDirs.delete(); // Deletes all local files. + remoteSessionRepo.close(); // Closes watchers and clears memory. + applicationRepo.close(); // Closes watchers. + localSessionRepo.deleteAllSessions(); // Closes watchers, clears memory, and deletes some local files and ZK session state. + } + + /** Deletes the tenant tree from ZooKeeper (application and session status for the tenant) and triggers {@link #close()}. */ + void delete() { + curator.delete(path); // Deletes tenant ZK tree: applications and sessions. } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 1b37984abcc..861b2f91c8f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -98,9 +98,7 @@ public class TenantBuilder { private void createLocalSessionRepo() { if (localSessionRepo == null) { - localSessionRepo = new LocalSessionRepo(tenantFileSystemDirs, localSessionLoader, componentRegistry.getClock(), - componentRegistry.getConfigserverConfig().sessionLifetime(), - componentRegistry.getCurator()); + localSessionRepo = new LocalSessionRepo(tenant, componentRegistry, tenantFileSystemDirs, localSessionLoader); } } @@ -129,8 +127,7 @@ public class TenantBuilder { tenant, Collections.singletonList(componentRegistry.getReloadListener()), ConfigResponseFactory.create(componentRegistry.getConfigserverConfig()), - componentRegistry.getHostRegistries(), - componentRegistry.getCurator()); + componentRegistry); if (hostValidator == null) { this.hostValidator = impl; } @@ -149,12 +146,12 @@ public class TenantBuilder { private void createRemoteSessionRepo() { if (remoteSessionRepo == null) { - remoteSessionRepo = new RemoteSessionRepo(componentRegistry.getCurator(), - remoteSessionFactory, - reloadHandler, - tenant, - applicationRepo, - componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant))); + remoteSessionRepo = new RemoteSessionRepo(componentRegistry, + remoteSessionFactory, + reloadHandler, + tenant, + applicationRepo); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 53d01fdf933..ad2472add89 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -73,7 +74,8 @@ public class TenantRepository { private final Curator curator; private final MetricUpdater metricUpdater; - private final ExecutorService pathChildrenExecutor = Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(TenantRepository.class.getName())); + private final ExecutorService zkCacheExecutor; + private final StripedExecutor<TenantName> zkWatcherExecutor; private final ExecutorService bootstrapExecutor; private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); private final Optional<Curator.DirectoryCache> directoryCache; @@ -104,6 +106,8 @@ public class TenantRepository { this.curator = globalComponentRegistry.getCurator(); metricUpdater = globalComponentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(globalComponentRegistry.getTenantListener()); + this.zkCacheExecutor = globalComponentRegistry.getZkCacheExecutor(); + this.zkWatcherExecutor = globalComponentRegistry.getZkWatcherExecutor(); curator.framework().getConnectionStateListenable().addListener(this::stateChanged); curator.create(tenantsPath); @@ -112,7 +116,7 @@ public class TenantRepository { curator.create(vespaPath); if (useZooKeeperWatchForTenantChanges) { - this.directoryCache = Optional.of(curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, pathChildrenExecutor)); + this.directoryCache = Optional.of(curator.createDirectoryCache(tenantsPath.getAbsolute(), false, false, zkCacheExecutor)); this.directoryCache.get().start(); this.directoryCache.get().addListener(this::childEvent); } else { @@ -147,22 +151,19 @@ public class TenantRepository { return curator.getChildren(tenantsPath).stream().map(TenantName::from).collect(Collectors.toSet()); } - private synchronized void updateTenants() { + /** Public for testing. */ + public synchronized void updateTenants() { Set<TenantName> allTenants = readTenantsFromZooKeeper(curator); log.log(LogLevel.DEBUG, "Create tenants, tenants found in zookeeper: " + allTenants); - checkForRemovedTenants(allTenants); - allTenants.stream().filter(tenantName -> ! tenants.containsKey(tenantName)).forEach(this::createTenant); + for (TenantName tenantName : Set.copyOf(tenants.keySet())) + if ( ! allTenants.contains(tenantName)) + zkWatcherExecutor.execute(tenantName, () -> closeTenant(tenantName)); + for (TenantName tenantName : allTenants) + if ( ! tenants.containsKey(tenantName)) + zkWatcherExecutor.execute(tenantName, () -> createTenant(tenantName)); metricUpdater.setTenants(tenants.size()); } - private void checkForRemovedTenants(Set<TenantName> newTenants) { - for (TenantName tenantName : ImmutableSet.copyOf(tenants.keySet())) { - if (!newTenants.contains(tenantName)) { - deleteTenant(tenantName); - } - } - } - private void bootstrapTenants() { // Keep track of tenants created Map<TenantName, Future<?>> futures = new HashMap<>(); @@ -273,19 +274,25 @@ public class TenantRepository { * Removes the given tenant from ZooKeeper and filesystem. Assumes that tenant exists. * * @param name name of the tenant - * @return this TenantRepository instance */ - public synchronized TenantRepository deleteTenant(TenantName name) { + public synchronized void deleteTenant(TenantName name) { if (name.equals(DEFAULT_TENANT)) throw new IllegalArgumentException("Deleting 'default' tenant is not allowed"); + if ( ! tenants.containsKey(name)) + throw new IllegalArgumentException("Deleting '" + name + "' failed, tenant does not exist"); + log.log(LogLevel.INFO, "Deleting tenant '" + name + "'"); + tenants.get(name).delete(); + } + + public synchronized void closeTenant(TenantName name) { Tenant tenant = tenants.remove(name); - if (tenant == null) { - throw new IllegalArgumentException("Deleting '" + name + "' failed, tenant does not exist"); - } + if (tenant == null) + throw new IllegalArgumentException("Closing '" + name + "' failed, tenant does not exist"); + + log.log(LogLevel.INFO, "Closing tenant '" + name + "'"); notifyRemovedTenant(name); tenant.close(); - return this; } // For unit testing @@ -353,9 +360,10 @@ public class TenantRepository { public void close() { directoryCache.ifPresent(Curator.DirectoryCache::close); try { - pathChildrenExecutor.shutdown(); + zkCacheExecutor.shutdown(); checkForRemovedApplicationsService.shutdown(); - pathChildrenExecutor.awaitTermination(50, TimeUnit.SECONDS); + zkWatcherExecutor.shutdownAndWait(); + zkCacheExecutor.awaitTermination(50, TimeUnit.SECONDS); checkForRemovedApplicationsService.awaitTermination(50, TimeUnit.SECONDS); } catch (InterruptedException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index 1430475e486..2f9a5eb9277 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -1,39 +1,44 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; -import java.time.Clock; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; - import com.yahoo.component.Version; +import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.config.FileReference; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.NotFoundException; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationMapper; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; +import com.yahoo.vespa.config.server.application.VersionDoesNotExistException; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.application.VersionDoesNotExistException; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; +import java.time.Clock; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; + +import static java.util.stream.Collectors.toSet; + /** * A per tenant request handler, for handling reload (activate application) and getConfig requests for * a set of applications belonging to a tenant. @@ -58,15 +63,15 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host TenantName tenant, List<ReloadListener> reloadListeners, ConfigResponseFactory responseFactory, - HostRegistries hostRegistries, - Curator curator) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. + GlobalComponentRegistry registry) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. this.metrics = metrics; this.tenant = tenant; this.reloadListeners = List.copyOf(reloadListeners); this.responseFactory = responseFactory; this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); - this.hostRegistry = hostRegistries.createApplicationHostRegistry(tenant); - this.applications = TenantApplications.create(curator, this, tenant); + this.hostRegistry = registry.getHostRegistries().createApplicationHostRegistry(tenant); + this.applications = TenantApplications.create(registry, this, tenant); + } /** @@ -248,7 +253,14 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host } return applicationId; } - + + @Override + public Set<FileReference> listFileReferences(ApplicationId applicationId) { + return applicationMapper.listApplications(applicationId).stream() + .flatMap(app -> app.getModel().fileReferences().stream()) + .collect(toSet()); + } + @Override public void verifyHosts(ApplicationId key, Collection<String> newHosts) { hostRegistry.verifyHosts(key, newHosts); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java new file mode 100644 index 00000000000..eaa4916d8fc --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java @@ -0,0 +1,86 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.config.model.api.TlsSecrets; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorOperations; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; + +import java.util.Optional; + +/** + * TLS Secret keys for applications (used to retrieve actual certificate/key from secret store). Persisted in ZooKeeper. + * + * @author andreer + */ +public class TlsSecretsKeys { + + private final Path path; + private final SecretStore secretStore; + private final Curator curator; + + public TlsSecretsKeys(Curator curator, Path tenantPath, SecretStore secretStore) { + this.curator = curator; + this.path = tenantPath.append("tlsSecretsKeys/"); + this.secretStore = secretStore; + } + + public Optional<TlsSecrets> readTlsSecretsKeyFromZookeeper(ApplicationId application) { + try { + Optional<byte[]> data = curator.getData(tlsSecretsKeyOf(application)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + String tlsSecretsKey = new ObjectMapper().readValue(data.get(), new TypeReference<String>() {}); + return readFromSecretStore(Optional.ofNullable(tlsSecretsKey)); + } catch (Exception e) { + throw new RuntimeException("Error reading TLS secret key of " + application, e); + } + } + + public void writeTlsSecretsKeyToZooKeeper(ApplicationId application, String tlsSecretsKey) { + if (tlsSecretsKey == null) return; + try { + byte[] data = new ObjectMapper().writeValueAsBytes(tlsSecretsKey); + curator.set(tlsSecretsKeyOf(application), data); + } catch (Exception e) { + throw new RuntimeException("Could not write TLS secret key of " + application, e); + } + } + + public Optional<TlsSecrets> getTlsSecrets(Optional<String> secretKeyname, ApplicationId applicationId) { + if (secretKeyname == null || secretKeyname.isEmpty()) { + return readTlsSecretsKeyFromZookeeper(applicationId); + } + return readFromSecretStore(secretKeyname); + } + + private Optional<TlsSecrets> readFromSecretStore(Optional<String> secretKeyname) { + if(secretKeyname.isEmpty()) return Optional.empty(); + TlsSecrets tlsSecretParameters = TlsSecrets.MISSING; + try { + String cert = secretStore.getSecret(secretKeyname.get() + "-cert"); + String key = secretStore.getSecret(secretKeyname.get() + "-key"); + tlsSecretParameters = new TlsSecrets(cert, key); + } catch (RuntimeException e) { + // Assume not ready yet +// log.log(LogLevel.DEBUG, "Could not fetch certificate/key with prefix: " + secretKeyname.get(), e); + } + return Optional.of(tlsSecretParameters); + } + + /** Returns a transaction which deletes these tls secrets key if they exist */ + public CuratorTransaction delete(ApplicationId application) { + if (!curator.exists(tlsSecretsKeyOf(application))) return CuratorTransaction.empty(curator); + return CuratorTransaction.from(CuratorOperations.delete(tlsSecretsKeyOf(application).getAbsolute()), curator); + } + + /** Returns the path storing the tls secrets key for an application */ + private Path tlsSecretsKeyOf(ApplicationId application) { + return path.append(application.serializedForm()); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 5c40f592a77..e013244c80c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -3,19 +3,20 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.common.base.Joiner; import com.yahoo.component.Version; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -69,7 +70,7 @@ public class ZKApplicationPackage implements ApplicationPackage { */ private AllocatedHosts readAllocatedHosts(String allocatedHostsPath, Optional<NodeFlavors> nodeFlavors) { try { - return AllocatedHosts.fromJson(liveApp.getBytes(allocatedHostsPath), nodeFlavors); + return AllocatedHostsSerializer.fromJson(liveApp.getBytes(allocatedHostsPath), nodeFlavors); } catch (Exception e) { throw new RuntimeException("Unable to read allocated hosts", e); } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 8ba505f213d..eef57f2ac9c 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -1,13 +1,13 @@ <?xml version="1.0" encoding="utf-8" ?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0" xmlns:preprocess="properties"> - <jdisc id="configserver" jetty="true" version="1.0"> + <container id="configserver" jetty="true" version="1.0"> <config name="container.jdisc.config.health-monitor"> <initialStatus>initializing</initialStatus> </config> <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" compressOnRotation="true" symlinkName="access.log" /> - <preprocess:include file='access-logging.xml' required='false' /> + <accesslog type="json" fileNamePattern="logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S" symlinkName="access-json.log" compressOnRotation="true" /> <component id="com.yahoo.vespa.config.server.ConfigServerBootstrap" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.monitoring.Metrics" bundle="configserver" /> @@ -38,6 +38,9 @@ <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.filedistribution.FileServer" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.maintenance.ConfigServerMaintenance" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.rpc.RpcRequestHandlerProvider" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.rpc.security.DummyNodeIdentifierProvider" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.rpc.security.DefaultRpcAuthorizerProvider" bundle="configserver" /> <component id="com.yahoo.vespa.serviceview.ConfigServerLocation" bundle="configserver" /> @@ -47,14 +50,14 @@ <preprocess:include file='config-models.xml' required='false' /> <preprocess:include file='node-repository.xml' required='false' /> - <preprocess:include file='hosted-vespa/routing-status.xml' required='false' /> + <preprocess:include file='routing-status.xml' required='false' /> <preprocess:include file='model-integration.xml' required='true' /> <component id="com.yahoo.vespa.configserver.flags.ConfigServerFlagSource" bundle="configserver-flags"/> <component id="com.yahoo.vespa.configserver.flags.db.FlagsDbImpl" bundle="configserver-flags"/> - <preprocess:include file='hosted-vespa/metrics-packets.xml' required='false' /> - <preprocess:include file='controller/container.xml' required='false' /> + <preprocess:include file='metrics-packets.xml' required='false' /> + <preprocess:include file='container.xml' required='false' /> <component id="com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.health.HealthMonitorManager" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.manager.UnionMonitorManager" bundle="service-monitor" /> @@ -153,8 +156,8 @@ <http> <server port="19071" id="configserver" /> - <preprocess:include file='hosted-vespa/http-server.xml' required='false' /> - <preprocess:include file='controller/http.xml' required='false' /> + <preprocess:include file='http-server.xml' required='false' /> + <preprocess:include file='http.xml' required='false' /> </http> <preprocess:include file='athenz-identity-provider.xml' required='false' /> @@ -164,5 +167,5 @@ <preprocess:include file='configserver-components.xml' required='false' /> <preprocess:include file='zookeeper-server-config.xml' required='false' /> - </jdisc> + </container> </services> diff --git a/configserver/src/main/sh/ping-configserver b/configserver/src/main/sh/ping-configserver deleted file mode 100755 index f899a3ddc62..00000000000 --- a/configserver/src/main/sh/ping-configserver +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -host=$1 -port=$2 -curl -s -S -m 5 http://$host:$port/state/v1/health | grep "status\": {\"code\": \"up\"}" -if [ $? -gt 0 ]; then - exit 1 -else - exit 0 -fi diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 12895465c6a..12683b1856a 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -135,7 +135,7 @@ vespa-run-as-vespa-user ${VESPA_HOME}/libexec/vespa/start-logd appdir="${VESPA_HOME}/conf/configserver-app" pidfile="${VESPA_HOME}/var/run/configserver.pid" -cfpfile="${VESPA_HOME}/var/jdisc_core/configserver.properties" +cfpfile="${VESPA_HOME}/var/jdisc_container/configserver.properties" bundlecachedir="${VESPA_HOME}/var/vespa/bundlecache/configserver" export JAVAVM_LD_PRELOAD= diff --git a/configserver/src/main/sh/vespa-configserver-remove-state b/configserver/src/main/sh/vespa-configserver-remove-state index faac37d48d4..c5cee479388 100755 --- a/configserver/src/main/sh/vespa-configserver-remove-state +++ b/configserver/src/main/sh/vespa-configserver-remove-state @@ -80,6 +80,9 @@ usage() { echo "The following options are recognized:" echo "" + echo "-h|-help) print this help text" + echo "-nosudo do not use sudo when running command" + echo "-sudo use sudo when running command" echo "-force do not ask for confirmation before removal" ) >&2 } @@ -101,7 +104,7 @@ while [ $# -gt 0 ]; do -h|-help) usage; exit 0;; -nosudo) shift; sudo="" ;; -sudo) shift; sudo="sudo" ;; - -force) shift; ask=false ;; + -force) shift; ask=false ;; *) echo "Unrecognized option '$1'" >&2; usage; exit 1;; esac done diff --git a/configserver/src/test/apps/app-jdisc-only-restart/services.xml b/configserver/src/test/apps/app-jdisc-only-restart/services.xml index b864ea206ef..5af96111f25 100644 --- a/configserver/src/test/apps/app-jdisc-only-restart/services.xml +++ b/configserver/src/test/apps/app-jdisc-only-restart/services.xml @@ -9,7 +9,7 @@ </slobroks> </admin> - <jdisc version="1.0"> + <container version="1.0"> <document-processing compressdocuments="true"> <chain id="ContainerWrapperTest"> <documentprocessor id="com.yahoo.vespa.config.AppleDocProc"/> @@ -24,6 +24,6 @@ <node hostalias="node1" /> </nodes> - </jdisc> + </container> </services> diff --git a/configserver/src/test/apps/app-jdisc-only/services.xml b/configserver/src/test/apps/app-jdisc-only/services.xml index 755ca1fd585..92a2c0d249e 100644 --- a/configserver/src/test/apps/app-jdisc-only/services.xml +++ b/configserver/src/test/apps/app-jdisc-only/services.xml @@ -2,7 +2,7 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version="1.0"> - <jdisc version="1.0"> + <container version="1.0"> <document-processing compressdocuments="true"> <chain id="ContainerWrapperTest"> <documentprocessor id="com.yahoo.vespa.config.AppleDocProc"/> @@ -17,6 +17,6 @@ <node hostalias="node1" /> </nodes> - </jdisc> + </container> </services> diff --git a/configserver/src/test/apps/app/services.xml b/configserver/src/test/apps/app/services.xml index 457a3fad397..509d7786be0 100644 --- a/configserver/src/test/apps/app/services.xml +++ b/configserver/src/test/apps/app/services.xml @@ -18,7 +18,7 @@ </content> - <jdisc version="1.0"> + <container version="1.0"> <document-processing compressdocuments="true"> <chain id="ContainerWrapperTest"> <documentprocessor id="com.yahoo.vespa.config.AppleDocProc"/> @@ -33,6 +33,6 @@ <node hostalias="node1" /> </nodes> - </jdisc> + </container> </services> diff --git a/configserver/src/test/apps/hosted-no-write-access-control/services.xml b/configserver/src/test/apps/hosted-no-write-access-control/services.xml index c2257ab34f7..e3262f6eeec 100644 --- a/configserver/src/test/apps/hosted-no-write-access-control/services.xml +++ b/configserver/src/test/apps/hosted-no-write-access-control/services.xml @@ -6,13 +6,13 @@ <nodes count='1'/> </admin> - <jdisc version="1.0"> + <container version="1.0"> <http> <server id="foo" port="4080" /> </http> <search/> <nodes count='1'/> - </jdisc> + </container> <content id="music" version="1.0"> <redundancy>1</redundancy> diff --git a/configserver/src/test/apps/hosted-routing-app/services.xml b/configserver/src/test/apps/hosted-routing-app/services.xml index 3d6680b8165..7eaf810ad25 100644 --- a/configserver/src/test/apps/hosted-routing-app/services.xml +++ b/configserver/src/test/apps/hosted-routing-app/services.xml @@ -6,7 +6,7 @@ <nodes count='1'/> </admin> - <jdisc version="1.0"> + <container version="1.0"> <http> <filtering> <access-control domain="foo" write="true" /> @@ -15,7 +15,7 @@ </http> <search/> <nodes type='host'/> - </jdisc> + </container> </services> diff --git a/configserver/src/test/apps/hosted/services.xml b/configserver/src/test/apps/hosted/services.xml index 7c2920958a2..d84718cf141 100644 --- a/configserver/src/test/apps/hosted/services.xml +++ b/configserver/src/test/apps/hosted/services.xml @@ -15,10 +15,6 @@ </http> <search/> <nodes count='1'/> - <rotations> - <rotation id='us-cluster'/> - <rotation id='eu-cluster'/> - </rotations> </container> <content id="music" version="1.0"> diff --git a/configserver/src/test/apps/validationOverride/services.xml b/configserver/src/test/apps/validationOverride/services.xml index 8dab1acf74c..bec088d3922 100644 --- a/configserver/src/test/apps/validationOverride/services.xml +++ b/configserver/src/test/apps/validationOverride/services.xml @@ -1,5 +1,5 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<jdisc version="1.0"> +<container version="1.0"> <http> <filtering> <access-control domain="foo" write="true" /> @@ -8,4 +8,4 @@ </http> <search/> <nodes count="2"/> -</jdisc> +</container> diff --git a/configserver/src/test/apps/zkapp/services.xml b/configserver/src/test/apps/zkapp/services.xml index cdbd6166c12..672f058bc0a 100644 --- a/configserver/src/test/apps/zkapp/services.xml +++ b/configserver/src/test/apps/zkapp/services.xml @@ -10,13 +10,13 @@ </slobroks> </admin> - <jdisc version="1.0"> + <container version="1.0"> <documentapi/> <search/> <nodes> <node hostalias="node1"/> </nodes> - </jdisc> + </container> <content version="1.0"> <redundancy>1</redundancy> diff --git a/configserver/src/test/apps/zkfeed/services.xml b/configserver/src/test/apps/zkfeed/services.xml index 943adc4b5a5..243d4e15d57 100644 --- a/configserver/src/test/apps/zkfeed/services.xml +++ b/configserver/src/test/apps/zkfeed/services.xml @@ -7,7 +7,7 @@ <logserver hostalias="node1" /> </admin> - <jdisc version="1.0"> + <container version="1.0"> <search> <include dir='dir1'/> <include dir='nested/dir2'/> @@ -16,6 +16,6 @@ <nodes> <node hostalias="node1" /> </nodes> - </jdisc> + </container> </services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index e0fa760b35d..9b76c349259 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; -import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; @@ -21,6 +20,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.deploy.DeployTester; +import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; @@ -37,6 +37,8 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -46,17 +48,11 @@ import java.util.Collections; import java.util.Optional; import java.util.Set; -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -83,21 +79,22 @@ public class ApplicationRepositoryTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Rule - public final WireMockRule wireMock = new WireMockRule(options().port(8080), true); - @Before public void setup() { Curator curator = new MockCurator(); tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() .curator(curator) .build()); + tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); tenantRepository.addTenant(tenant3); orchestrator = new OrchestratorMock(); provisioner = new SessionHandlerTest.MockProvisioner(); - applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, clock); + applicationRepository = new ApplicationRepository(tenantRepository, + provisioner, + orchestrator, + clock); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } @@ -151,7 +148,11 @@ public class ApplicationRepositoryTest { @Test public void getLogs() { - wireMock.stubFor(get(urlEqualTo("/logs")).willReturn(aResponse().withStatus(200))); + applicationRepository = new ApplicationRepository(tenantRepository, + provisioner, + orchestrator, + new MockLogRetriever(), + clock); deployApp(testAppLogServerWithContainer); HttpResponse response = applicationRepository.getLogs(applicationId(), Optional.empty(), ""); assertEquals(200, response.getStatus()); @@ -159,14 +160,24 @@ public class ApplicationRepositoryTest { @Test public void getLogsForHostname() { - wireMock.stubFor(get(urlEqualTo("/logs")).willReturn(aResponse().withStatus(200))); - deployApp(testAppLogServerWithContainer); - HttpResponse response = applicationRepository.getLogs(applicationId(), Optional.of("localhost"), ""); + applicationRepository = new ApplicationRepository(tenantRepository, + provisioner, + orchestrator, + new MockLogRetriever(), + clock); + ApplicationId applicationId = ApplicationId.from("hosted-vespa", "tenant-host", "default"); + deployApp(testAppLogServerWithContainer, new PrepareParams.Builder().applicationId(applicationId).build()); + HttpResponse response = applicationRepository.getLogs(applicationId, Optional.of("localhost"), ""); assertEquals(200, response.getStatus()); } @Test(expected = IllegalArgumentException.class) public void refuseToGetLogsFromHostnameNotInApplication() { + applicationRepository = new ApplicationRepository(tenantRepository, + provisioner, + orchestrator, + new MockLogRetriever(), + clock); deployApp(testAppLogServerWithContainer); HttpResponse response = applicationRepository.getLogs(applicationId(), Optional.of("host123.fake.yahoo.com"), ""); assertEquals(200, response.getStatus()); @@ -273,8 +284,8 @@ public class ApplicationRepositoryTest { assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); assertNull(tenant.getRemoteSessionRepo().getSession(sessionId)); assertTrue(provisioner.removed); - assertThat(provisioner.lastApplicationId.tenant(), is(tenant.getName())); - assertThat(provisioner.lastApplicationId, is(applicationId())); + assertEquals(tenant.getName(), provisioner.lastApplicationId.tenant()); + assertEquals(applicationId(), provisioner.lastApplicationId); assertFalse(applicationRepository.delete(applicationId())); } @@ -292,7 +303,7 @@ public class ApplicationRepositoryTest { // Delete app with id fooId, should not affect original app assertTrue(applicationRepository.delete(fooId)); - assertThat(provisioner.lastApplicationId, is(fooId)); + assertEquals(fooId, provisioner.lastApplicationId); assertNotNull(applicationRepository.getActiveSession(applicationId())); assertTrue(applicationRepository.delete(applicationId())); @@ -332,7 +343,7 @@ public class ApplicationRepositoryTest { // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions tester.applicationRepository().deleteExpiredLocalSessions(); - final Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); + Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions(); assertEquals(1, sessions.size()); assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId()); @@ -372,4 +383,28 @@ public class ApplicationRepositoryTest { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); return applicationRepository.getMetadataFromSession(tenant, sessionId); } + + private static class MockLogRetriever extends LogRetriever { + + @Override + public HttpResponse getLogs(String logServerHostname) { + return new MockHttpResponse(); + } + + private static class MockHttpResponse extends HttpResponse { + + private MockHttpResponse() { + super(200); + } + + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log line".getBytes(StandardCharsets.UTF_8)); + } + + } + + + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 2e7b5a1f4d9..0b56591d6a1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -61,7 +60,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3"); + InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); DeployTester tester = new DeployTester(configserverConfig, provisioner); tester.deployApp("src/test/apps/hosted/"); @@ -94,7 +93,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWithVipStatusFile() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3"); + InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); DeployTester tester = new DeployTester(configserverConfig, provisioner); tester.deployApp("src/test/apps/hosted/"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index 476f77ae1db..e4ff8702ff1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -14,6 +14,8 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.rpc.RpcServer; +import com.yahoo.vespa.config.server.rpc.RpcRequestHandlerProvider; +import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.session.SessionTest; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -66,7 +68,7 @@ public class InjectedGlobalComponentRegistryTest { .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath())); sessionPreparer = new SessionTest.MockSessionPreparer(); rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(temporaryFolder.newFolder("filereferences"))); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(temporaryFolder.newFolder("filereferences")), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); generationCounter = new SuperModelGenerationCounter(curator); defRepo = new StaticConfigDefinitionRepo(); permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); @@ -76,7 +78,7 @@ public class InjectedGlobalComponentRegistryTest { globalComponentRegistry = new InjectedGlobalComponentRegistry(curator, configCurator, metrics, modelFactoryRegistry, sessionPreparer, rpcServer, configserverConfig, generationCounter, defRepo, permanentApplicationPackage, hostRegistries, hostProvisionerProvider, zone, - new ConfigServerDB(configserverConfig), new InMemoryFlagSource()); + new ConfigServerDB(configserverConfig), new InMemoryFlagSource(), new MockSecretStore()); } @Test @@ -88,7 +90,6 @@ public class InjectedGlobalComponentRegistryTest { assertThat(globalComponentRegistry.getConfigserverConfig(), is(configserverConfig)); assertThat(globalComponentRegistry.getReloadListener().hashCode(), is(rpcServer.hashCode())); assertThat(globalComponentRegistry.getTenantListener().hashCode(), is(rpcServer.hashCode())); - assertThat(globalComponentRegistry.getSuperModelGenerationCounter(), is(generationCounter)); assertThat(globalComponentRegistry.getStaticConfigDefinitionRepo(), is(defRepo)); assertThat(globalComponentRegistry.getPermanentApplicationPackage(), is(permanentApplicationPackage)); assertThat(globalComponentRegistry.getHostRegistries(), is(hostRegistries)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java new file mode 100644 index 00000000000..8a77b53875e --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java @@ -0,0 +1,35 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server; + +import com.yahoo.container.jdisc.secretstore.SecretStore; + +import java.util.HashMap; +import java.util.Map; + +public class MockSecretStore implements SecretStore { + Map<String, String> secrets = new HashMap<>(); + + @Override + public String getSecret(String key) { + if(secrets.containsKey(key)) + return secrets.get(key); + throw new RuntimeException("Key not found: " + key); + } + + @Override + public String getSecret(String key, int version) { + return getSecret(key); + } + + public void put(String key, String value) { + secrets.put(key, value); + } + + public void remove(String key) { + secrets.remove(key); + } + + public void clear() { + secrets.clear(); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 1b4ed2283ba..860bbdc134c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.component.Version; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -14,6 +15,7 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Test; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -33,6 +35,10 @@ public class ModelContextImplTest { final Rotation rotation = new Rotation("this.is.a.mock.rotation"); final Set<Rotation> rotations = Collections.singleton(rotation); + + final ContainerEndpoint endpoint = new ContainerEndpoint("foo", List.of("a", "b")); + final Set<ContainerEndpoint> endpoints = Collections.singleton(endpoint); + final InMemoryFlagSource flagSource = new InMemoryFlagSource(); ModelContext context = new ModelContextImpl( @@ -53,9 +59,11 @@ public class ModelContextImplTest { false, Zone.defaultZone(), rotations, + endpoints, false, false, - flagSource), + flagSource, + null), Optional.empty(), new Version(6), new Version(6)); @@ -71,7 +79,8 @@ public class ModelContextImplTest { assertNotNull(context.properties().zone()); assertFalse(context.properties().hostedVespa()); assertThat(context.properties().rotations(), equalTo(rotations)); + assertThat(context.properties().endpoints(), equalTo(endpoints)); assertThat(context.properties().isFirstTimeDeployment(), equalTo(false)); - assertThat(context.properties().useDedicatedNodeForLogserver(), equalTo(false)); + assertThat(context.properties().useDedicatedNodeForLogserver(), equalTo(true)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 35ad97b7b43..f7b900c8f02 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -56,17 +56,16 @@ public class SuperModelRequestHandlerTest { ApplicationId bar = applicationId("a", "foo"); assertNotNull(controller.getHandler()); - long gen = counter.increment(); + long gen = counter.get(); controller.reloadConfig(createApp(foo, 3l)); assertNotNull(controller.getHandler()); - assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getGeneration(), is(gen + 1)); controller.reloadConfig(createApp(foo, 4l)); - assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getGeneration(), is(gen + 2)); // Test that a new app is used when there already exist an application with the same id assertThat(controller.getHandler().getSuperModel().applicationModels().get(foo).getGeneration(), is(4l)); - gen = counter.increment(); controller.reloadConfig(createApp(bar, 2l)); - assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getGeneration(), is(gen + 3)); } @Test @@ -75,22 +74,21 @@ public class SuperModelRequestHandlerTest { ApplicationId bar = applicationId("a", "bar"); ApplicationId baz = applicationId("b", "baz"); - long gen = counter.increment(); + long gen = counter.get(); controller.reloadConfig(createApp(foo, 3l)); controller.reloadConfig(createApp(bar, 30l)); controller.reloadConfig(createApp(baz, 9l)); - assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getGeneration(), is(gen + 3)); assertThat(controller.getHandler().getSuperModel().applicationModels().size(), is(3)); assertEquals(Arrays.asList(foo, bar, baz), new ArrayList<>(controller.getHandler().getSuperModel().applicationModels().keySet())); controller.removeApplication(new ApplicationId.Builder().tenant("a").applicationName("unknown").build()); - assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getGeneration(), is(gen + 4)); assertThat(controller.getHandler().getSuperModel().applicationModels().size(), is(3)); assertEquals(Arrays.asList(foo, bar, baz), new ArrayList<>(controller.getHandler().getSuperModel().applicationModels().keySet())); - gen = counter.increment(); controller.removeApplication(bar); assertThat(controller.getHandler().getSuperModel().applicationModels().size(), is(2)); assertEquals(Arrays.asList(foo, baz), new ArrayList<>(controller.getHandler().getSuperModel().applicationModels().keySet())); - assertThat(controller.getHandler().getGeneration(), is(gen)); + assertThat(controller.getHandler().getGeneration(), is(gen + 5)); } @Test @@ -101,9 +99,9 @@ public class SuperModelRequestHandlerTest { manager = new SuperModelManager(configserverConfig, emptyNodeFlavors(), counter, new InMemoryFlagSource()); controller = new SuperModelRequestHandler(new TestConfigDefinitionRepo(), configserverConfig, manager); - long gen = counter.increment(); + long gen = counter.get(); controller.reloadConfig(createApp(foo, 3L)); - assertThat(controller.getHandler().getGeneration(), is(masterGen + gen)); + assertThat(controller.getHandler().getGeneration(), is(masterGen + gen + 1)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index ef1f0f380e3..a304f74858b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -3,10 +3,14 @@ package com.yahoo.vespa.config.server; import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.InThreadExecutorService; +import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.Provisioner; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; @@ -28,6 +32,7 @@ import com.yahoo.vespa.model.VespaModelFactory; import java.time.Clock; import java.util.Collections; import java.util.Optional; +import java.util.concurrent.ExecutorService; /** @@ -40,7 +45,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final Metrics metrics; private final SessionPreparer sessionPreparer; private final ConfigserverConfig configserverConfig; - private final SuperModelGenerationCounter superModelGenerationCounter; private final ConfigDefinitionRepo defRepo; private final ReloadListener reloadListener; private final TenantListener tenantListener; @@ -52,6 +56,9 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final Zone zone; private final Clock clock; private final ConfigServerDB configServerDB; + private final StripedExecutor<TenantName> zkWatcherExecutor; + private final ExecutorService zkCacheExecutor; + private final SecretStore secretStore; private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, @@ -65,14 +72,14 @@ public class TestComponentRegistry implements GlobalComponentRegistry { ReloadListener reloadListener, TenantListener tenantListener, Zone zone, - Clock clock) { + Clock clock, + SecretStore secretStore) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; this.configserverConfig = configserverConfig; this.reloadListener = reloadListener; this.tenantListener = tenantListener; - this.superModelGenerationCounter = new SuperModelGenerationCounter(curator); this.defRepo = defRepo; this.permanentApplicationPackage = permanentApplicationPackage; this.hostRegistries = hostRegistries; @@ -83,6 +90,9 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.zone = zone; this.clock = clock; this.configServerDB = new ConfigServerDB(configserverConfig); + this.zkWatcherExecutor = new StripedExecutor<>(new InThreadExecutorService()); + this.zkCacheExecutor = new InThreadExecutorService(); + this.secretStore = secretStore; } public static class Builder { @@ -92,6 +102,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private ConfigserverConfig configserverConfig = new ConfigserverConfig( new ConfigserverConfig.Builder() .configServerDBDir(Files.createTempDir().getAbsolutePath()) + .sessionLifetime(5) .configDefinitionsDir(Files.createTempDir().getAbsolutePath())); private ConfigDefinitionRepo defRepo = new StaticConfigDefinitionRepo(); private TenantRequestHandlerTest.MockReloadListener reloadListener = new TenantRequestHandlerTest.MockReloadListener(); @@ -151,14 +162,15 @@ public class TestComponentRegistry implements GlobalComponentRegistry { .orElse(new MockFileDistributionFactory(configserverConfig)); HostProvisionerProvider hostProvisionerProvider = hostProvisioner. map(HostProvisionerProvider::withProvisioner).orElseGet(HostProvisionerProvider::empty); + SecretStore secretStore = new MockSecretStore(); SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionFactory, hostProvisionerProvider, permApp, configserverConfig, defRepo, curator, - zone, new InMemoryFlagSource()); + zone, new InMemoryFlagSource(), secretStore); return new TestComponentRegistry(curator, ConfigCurator.create(curator), metrics, modelFactoryRegistry, permApp, fileDistributionFactory, hostRegistries, configserverConfig, sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, - zone, clock); + zone, clock, secretStore); } } @@ -177,8 +189,6 @@ public class TestComponentRegistry implements GlobalComponentRegistry { @Override public ReloadListener getReloadListener() { return reloadListener; } @Override - public SuperModelGenerationCounter getSuperModelGenerationCounter() { return superModelGenerationCounter; } - @Override public ConfigDefinitionRepo getStaticConfigDefinitionRepo() { return defRepo; } @Override public PermanentApplicationPackage getPermanentApplicationPackage() { return permanentApplicationPackage; } @@ -198,9 +208,25 @@ public class TestComponentRegistry implements GlobalComponentRegistry { public Clock getClock() { return clock;} @Override public ConfigServerDB getConfigServerDB() { return configServerDB;} + + @Override + public StripedExecutor<TenantName> getZkWatcherExecutor() { + return zkWatcherExecutor; + } + @Override public FlagSource getFlagSource() { return new InMemoryFlagSource(); } + @Override + public ExecutorService getZkCacheExecutor() { + return zkCacheExecutor; + } + + @Override + public SecretStore getSecretStore() { + return secretStore; + } + public FileDistributionFactory getFileDistributionFactory() { return fileDistributionFactory; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 7faf33b2ff2..496da2cf809 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -1,8 +1,9 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; +import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -26,7 +27,6 @@ import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen - * @since 5.1 */ public class CompressedApplicationInputStreamTest { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index ac7ff1e85c5..e3335dded4c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.MockReloadHandler; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -115,7 +116,7 @@ public class TenantApplicationsTest { } private TenantApplications createZKAppRepo(MockReloadHandler reloadHandler) { - return TenantApplications.create(curator, reloadHandler, tenantName); + return TenantApplications.create(new TestComponentRegistry.Builder().curator(curator).build(), reloadHandler, tenantName); } private static ApplicationId createApplicationId(String name) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 6b67dcc4e9a..f619bd92bef 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.http.LogRetriever; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -126,8 +127,9 @@ public class DeployTester { applicationRepository = new ApplicationRepository(tenantRepository, new ProvisionerAdapter(provisioner), new OrchestratorMock(), - clock, - configserverConfig); + configserverConfig, + new LogRetriever(), + clock); } public Tenant tenant() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index c5d1e8dc0a1..b363c749212 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -15,11 +15,8 @@ import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.configchange.MockRestartAction; @@ -39,7 +36,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -97,7 +93,7 @@ public class HostedDeployTest { DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, Zone.defaultZone()); tester.deployApp("src/test/apps/hosted/", "6.2.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } /** @@ -108,7 +104,8 @@ public class HostedDeployTest { public void testCreateOnlyNeededModelVersions() { List<Host> hosts = Arrays.asList(createHost("host1", "6.0.0"), createHost("host2", "6.1.0"), - createHost("host3")); //Use a host with no version as well + createHost("host3"), //Use a host with no version as well + createHost("host4", "6.1.0")); InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); @@ -127,7 +124,7 @@ public class HostedDeployTest { DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); // Deploy with version that does not exist on hosts, the model for this version should also be created tester.deployApp("src/test/apps/hosted/", "7.0.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory600.creationCount() > 0); @@ -145,7 +142,7 @@ public class HostedDeployTest { */ @Test public void testCreateOnlyNeededModelVersionsNewNodes() { - List<Host> hosts = Arrays.asList(createHost("host1"), createHost("host2"), createHost("host3")); + List<Host> hosts = Arrays.asList(createHost("host1"), createHost("host2"), createHost("host3"), createHost("host4")); InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); @@ -157,7 +154,7 @@ public class HostedDeployTest { DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); // Deploy with version that does not exist on hosts, the model for this version should also be created tester.deployApp("src/test/apps/hosted/", "7.0.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory700.creationCount() > 0); @@ -195,7 +192,8 @@ public class HostedDeployTest { String newestOnNewMajorVersion = newestMajorVersion + ".2.0"; List<Host> hosts = Arrays.asList(createHost("host1", oldestVersion), createHost("host2", newestOnOldMajorVersion), - createHost("host3", newestOnOldMajorVersion)); + createHost("host3", newestOnOldMajorVersion), + createHost("host4", newestOnOldMajorVersion)); InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); CountingModelFactory factory1 = DeployTester.createModelFactory(Version.fromString(oldestVersion)); @@ -205,7 +203,7 @@ public class HostedDeployTest { DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); tester.deployApp("src/test/apps/hosted/", oldestVersion, Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory1.creationCount() > 0); @@ -256,7 +254,7 @@ public class HostedDeployTest { @Test public void testAccessControlIsOnlyCheckedWhenNoProdDeploymentExists() { // Provisioner does not reuse hosts, so need twice as many hosts as app requires - List<Host> hosts = IntStream.rangeClosed(1,6).mapToObj(i -> createHost("host" + i, "6.0.0")).collect(Collectors.toList()); + List<Host> hosts = IntStream.rangeClosed(1,8).mapToObj(i -> createHost("host" + i, "6.0.0")).collect(Collectors.toList()); InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); @@ -269,12 +267,12 @@ public class HostedDeployTest { ApplicationId applicationId = tester.applicationId(); // Deploy with oldest version tester.deployApp("src/test/apps/hosted/", "6.0.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(applicationId).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(applicationId).getHosts().size()); // Deploy with version that does not exist on hosts and with app package that has no write access control, // validation of access control should not be done, since the app is already deployed in prod tester.deployApp("src/test/apps/hosted-no-write-access-control", "6.1.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(applicationId).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(applicationId).getHosts().size()); } @Test @@ -321,7 +319,8 @@ public class HostedDeployTest { public void testThatConfigChangeActionsAreCollectedFromAllModels() { List<Host> hosts = Arrays.asList(createHost("host1", "6.1.0"), createHost("host2", "6.2.0"), - createHost("host3", "6.2.0")); + createHost("host3", "6.2.0"), + createHost("host4", "6.2.0")); InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); List<ServiceInfo> services = Collections.singletonList( new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); @@ -333,30 +332,12 @@ public class HostedDeployTest { DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.2.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); List<RestartActions.Entry> actions = prepareResult.configChangeActions().getRestartActions().getEntries(); assertThat(actions.size(), is(1)); assertThat(actions.get(0).getMessages(), equalTo(ImmutableSet.of("change", "other change"))); } - @Test - public void testDeployWithClusterRotations() { - CountingModelFactory modelFactory = DeployTester.createModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(Collections.singletonList(modelFactory), createConfigserverConfig()); - ApplicationId applicationId = tester.applicationId(); - - tester.deployApp("src/test/apps/hosted/", "4.5.6", Instant.now()); - Set<HostSpec> containers = tester.getAllocatedHostsOf(applicationId).getHosts().stream() - .filter(h -> h.membership().get().cluster().type() == ClusterSpec.Type.container) - .collect(Collectors.toSet()); - assertFalse("Allocated container hosts", containers.isEmpty()); - - Set<RotationName> expected = Set.of(RotationName.from("eu-cluster"), RotationName.from("us-cluster")); - for (HostSpec container : containers) { - assertEquals(expected, container.membership().get().cluster().rotations()); - } - } - private static ConfigserverConfig createConfigserverConfig() { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(Files.createTempDir().getAbsolutePath()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index e01b2eccb35..918670d71f2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -13,9 +13,9 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.HostSpec; import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -30,10 +30,11 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.fromJson; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Unit tests for ZooKeeperClient. @@ -199,7 +200,7 @@ public class ZooKeeperClientTest { Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); assertTrue(zk.exists(hostsPath.getAbsolute())); - AllocatedHosts deserialized = AllocatedHosts.fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); + AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); assertEquals(hosts, deserialized.getHosts()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index b818c39d433..0381af57cc3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -99,7 +99,7 @@ public class SessionHandlerTest { } private MockSession(long id, ApplicationPackage app, InMemoryFlagSource flagSource) { - super(TenantName.defaultName(), id, null, new SessionContext(null, new MockSessionZKClient(MockApplicationPackage.createEmpty()), null, null, new HostRegistry<>(), null, flagSource)); + super(TenantName.defaultName(), id, null, new SessionContext(null, new MockSessionZKClient(MockApplicationPackage.createEmpty()), null, null, new HostRegistry<>(), flagSource)); this.app = app; this.preparer = new SessionTest.MockSessionPreparer(); this.flagSource = flagSource; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index a843212927b..41db57ab1e0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -197,7 +197,7 @@ public class ApplicationHandlerTest { ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant(mytenantName).build(); HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone); assertEquals(404, responseForUnknown.getStatus()); - assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: mytenant.unknown\"}", + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'mytenant.unknown'\"}", SessionHandlerTest.getRenderedString(responseForUnknown)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 858d1e0eaa7..d94194e58d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -105,7 +105,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(modelFactory))) .build(); tenantRepository = new TenantRepository(componentRegistry, false); - applicationRepo = TenantApplications.create(curator, new MockReloadHandler(), tenantName); + applicationRepo = TenantApplications.create(componentRegistry, new MockReloadHandler(), tenantName); localRepo = new LocalSessionRepo(clock, curator); pathPrefix = "/application/v2/tenant/" + tenantName + "/session/"; hostProvisioner = new MockProvisioner(); @@ -232,7 +232,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { ApplicationPackage app = FilesApplicationPackage.fromFileWithDeployData(testApp, deployData); localRepo.addSession(new LocalSession(tenantName, sessionId, new SessionTest.MockSessionPreparer(), new SessionContext(app, zkc, new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)), - applicationRepo, new HostRegistry<>(), new SuperModelGenerationCounter(curator), + applicationRepo, new HostRegistry<>(), flagSource))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 0946ef3992c..ac4b4ea9005 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; +import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -72,7 +72,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { @Before public void setupRepo() { Curator curator = new MockCurator(); - applicationRepo = TenantApplications.create(curator, new MockReloadHandler(), tenant); + applicationRepo = TenantApplications.create(componentRegistry, new MockReloadHandler(), tenant); localSessionRepo = new LocalSessionRepo(Clock.systemUTC(), curator); tenantRepository = new TenantRepository(componentRegistry, false); sessionFactory = new MockSessionFactory(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 11c0cf057cc..b6c3de8a1b1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -87,7 +87,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { .withSessionFactory(new MockSessionFactory()) .withLocalSessionRepo(localRepo) .withRemoteSessionRepo(remoteSessionRepo) - .withApplicationRepo(TenantApplications.create(curator, new MockReloadHandler(), tenant)); + .withApplicationRepo(TenantApplications.create(componentRegistry, new MockReloadHandler(), tenant)); tenantRepository.addTenant(tenantBuilder); } @@ -403,7 +403,6 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { null, null, new HostRegistry<>(), - null, null)); this.exception = exception; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java index a2f52bc5321..fdc6ffeacf0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -37,6 +37,7 @@ public class TenantsMaintainerTest { assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1)).run(); + tenantRepository.updateTenants(); // One tenant should now have been deleted assertNull(tenantRepository.getTenant(shouldBeDeleted)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 0e124addaf7..34a4074c7cf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; @@ -20,11 +21,14 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.xml.sax.SAXException; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -33,20 +37,34 @@ import java.util.Random; import java.util.Set; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; /** * @author Ulf Lilleengen */ +@RunWith(Parameterized.class) public class LbServicesProducerTest { private static final String rotation1 = "rotation-1"; private static final String rotation2 = "rotation-2"; private static final String rotationString = rotation1 + "," + rotation2; private static final Set<Rotation> rotations = Collections.singleton(new Rotation(rotationString)); + private static final Set<ContainerEndpoint> endpoints = Set.of( + new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2")) + ); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private final boolean useGlobalServiceId; + + @Parameterized.Parameters + public static Object[] useGlobalServiceId() { + return new Object[] { true, false }; + } + + public LbServicesProducerTest(boolean useGlobalServiceId) { + this.useGlobalServiceId = useGlobalServiceId; + } @Test public void testDeterministicGetConfig() throws IOException, SAXException { @@ -110,7 +128,7 @@ public class LbServicesProducerTest { Zone zone = new Zone(Environment.prod, regionName); Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() .zone(zone) - .properties(new TestProperties())); + .properties(new TestProperties().setHostedVespa(true))); return getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); } @@ -123,18 +141,40 @@ public class LbServicesProducerTest { @Test public void testConfigAliasesWithRotations() throws IOException, SAXException { - Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder().rotations(rotations)); + assumeTrue(useGlobalServiceId); + + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() + .rotations(rotations) + .properties(new TestProperties().setHostedVespa(true))); RegionName regionName = RegionName.from("us-east-1"); - LbServicesConfig conf = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel); - final LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName); - assertThat(services.servicealiases().size(), is(1)); - assertThat(services.endpointaliases().size(), is(4)); - assertThat(services.servicealiases(0), is("service1")); - assertThat(services.endpointaliases(0), is("foo1.bar1.com")); - assertThat(services.endpointaliases(1), is("foo2.bar2.com")); - assertThat(services.endpointaliases(2), is(rotation1)); - assertThat(services.endpointaliases(3), is(rotation2)); + var services = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel) + .tenants("foo") + .applications("foo:prod:" + regionName.value() + ":default") + .hosts("foo.foo.yahoo.com") + .services(QRSERVER.serviceName); + + assertThat(services.servicealiases(), contains("service1")); + assertThat("Missing rotations in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); + } + + @Test + public void testConfigAliasesWithEndpoints() throws IOException, SAXException { + assumeFalse(useGlobalServiceId); + + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder() + .endpoints(endpoints) + .properties(new TestProperties().setHostedVespa(true))); + RegionName regionName = RegionName.from("us-east-1"); + + var services = getLbServicesConfig(new Zone(Environment.prod, regionName), testModel) + .tenants("foo") + .applications("foo:prod:" + regionName.value() + ":default") + .hosts("foo.foo.yahoo.com") + .services(QRSERVER.serviceName); + + assertThat(services.servicealiases(), contains("service1")); + assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); } private Map<TenantName, Set<ApplicationInfo>> randomizeApplications(Map<TenantName, Set<ApplicationInfo>> testModel, int seed) { @@ -181,7 +221,7 @@ public class LbServicesProducerTest { private ApplicationPackage createApplicationPackage(String host1, String host2) { String hosts = "<hosts><host name='" + host1 + "'><alias>node1</alias></host><host name='" + host2 + "'><alias>node2</alias></host></hosts>"; String services = "<services><admin version='2.0'><adminserver hostalias='node1' /><logserver hostalias='node1' /><slobroks><slobrok hostalias='node1' /><slobrok hostalias='node2' /></slobroks></admin>" - + "<jdisc id='mydisc' version='1.0'>" + + + "<container id='mydisc' version='1.0'>" + " <aliases>" + " <endpoint-alias>foo2.bar2.com</endpoint-alias>" + " <service-alias>service1</service-alias>" + @@ -191,16 +231,34 @@ public class LbServicesProducerTest { " <node hostalias='node1' />" + " </nodes>" + " <search/>" + - "</jdisc>" + + "</container>" + "</services>"; - String deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + - "<deployment version='1.0'>" + - " <test />" + - " <prod global-service-id='mydisc'>" + - " <region active='true'>us-east-1</region>" + - " <region active='false'>us-east-2</region>" + - " </prod>" + - "</deployment>"; + + String deploymentInfo; + + if (useGlobalServiceId) { + deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod global-service-id='mydisc'>" + + " <region active='true'>us-east-1</region>" + + " <region active='false'>us-east-2</region>" + + " </prod>" + + "</deployment>"; + } else { + deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + + "<deployment version='1.0'>" + + " <test />" + + " <prod>" + + " <region active='true'>us-east-1</region>" + + " <region active='false'>us-east-2</region>" + + " </prod>" + + " <endpoints>" + + " <endpoint container-id='mydisc' />" + + " </endpoints>" + + "</deployment>"; + } + return new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withDeploymentSpec(deploymentInfo).build(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java index c6a607f81b1..7ce2c39c6a4 100755 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java @@ -77,7 +77,7 @@ public class RoutingProducerTest { private ApplicationPackage createApplicationPackage(String host1, String host2) { String hosts = "<hosts><host name='" + host1 + "'><alias>node1</alias></host><host name='" + host2 + "'><alias>node2</alias></host></hosts>"; String services = "<services><admin version='2.0'><adminserver hostalias='node1' /><logserver hostalias='node1' /><slobroks><slobrok hostalias='node1' /><slobrok hostalias='node2' /></slobroks></admin>" - + "<jdisc id='mydisc' version='1.0'>" + + + "<container id='mydisc' version='1.0'>" + " <aliases>" + " <endpoint-alias>foo2.bar2.com</endpoint-alias>" + " <service-alias>service1</service-alias>" + @@ -87,7 +87,7 @@ public class RoutingProducerTest { " <node hostalias='node1' />" + " </nodes>" + " <search/>" + - "</jdisc>" + + "</container>" + "</services>"; String deploymentInfo ="<?xml version='1.0' encoding='UTF-8'?>" + "<deployment version='1.0'>" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java index bfc06a58b16..0f6cd10d564 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.component.Version; import com.yahoo.vespa.config.ConfigKey; @@ -91,6 +92,11 @@ public class MockRequestHandler implements RequestHandler, ReloadHandler, Tenant } @Override + public Set<FileReference> listFileReferences(ApplicationId applicationId) { + return Set.of(); + } + + @Override public RequestHandler getRequestHandler() { return this; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 0f8bfa5068c..5fa51e1c404 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; import java.io.File; @@ -38,7 +39,7 @@ public class MockRpc extends RpcServer { public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication, File tempDir) { super(createConfig(port), null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(tempDir)); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(tempDir), new NoopRpcAuthorizer(), new RpcRequestHandlerProvider()); if (createDefaultTenant) { onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index c8bc9364922..086dfa5d0d3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -114,7 +114,6 @@ public class RpcServerTest { } private void testEnabled(RpcTester tester) throws IOException, SAXException { - tester.generationCounter().increment(); Application app = new Application(new VespaModel(MockApplicationPackage.createEmpty()), new ServerCache(), 2L, @@ -174,7 +173,7 @@ public class RpcServerTest { private void testPrintStatistics(RpcTester tester) { Request req = new Request("printStatistics"); - tester.rpcServer().printStatistics(req); + tester.performRequest(req); assertThat(req.returnValues().get(0).asString(), is("Delayed responses queue size: 0")); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index dd66f720b1f..53463585582 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer; import com.yahoo.vespa.config.server.tenant.MockTenantProvider; import com.yahoo.vespa.config.server.tenant.TenantHandlerProvider; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -45,13 +46,13 @@ public class RpcTester implements AutoCloseable { private final ManualClock clock = new ManualClock(Instant.ofEpochMilli(100)); private final String myHostname = HostName.getLocalhost(); private final HostLivenessTracker hostLivenessTracker = new ConfigRequestHostLivenessTracker(clock); + private final MockTenantProvider tenantProvider; + private final GenerationCounter generationCounter; + private final Spec spec; private RpcServer rpcServer; - private MockTenantProvider tenantProvider; - private GenerationCounter generationCounter; private Thread t; private Supervisor sup; - private Spec spec; private List<Integer> allocatedPorts; @@ -97,7 +98,9 @@ public class RpcTester implements AutoCloseable { generationCounter, new InMemoryFlagSource())), Metrics.createTestMetrics(), new HostRegistries(), - hostLivenessTracker, new FileServer(temporaryFolder.newFolder())); + hostLivenessTracker, new FileServer(temporaryFolder.newFolder()), + new NoopRpcAuthorizer(), + new RpcRequestHandlerProvider()); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); t.start(); @@ -145,7 +148,4 @@ public class RpcTester implements AutoCloseable { return tenantProvider; } - GenerationCounter generationCounter() { - return generationCounter; - } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java new file mode 100644 index 00000000000..a1d4f28cb74 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizerTest.java @@ -0,0 +1,324 @@ +package com.yahoo.vespa.config.server.rpc.security;// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.cloud.config.LbServicesConfig; +import com.yahoo.cloud.config.RoutingConfig; +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.FileReference; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.security.NodeIdentifier; +import com.yahoo.config.provision.security.NodeIdentifierException; +import com.yahoo.config.provision.security.NodeIdentity; +import com.yahoo.jrt.Request; +import com.yahoo.jrt.SecurityContext; +import com.yahoo.jrt.StringValue; +import com.yahoo.jrt.Target; +import com.yahoo.jrt.Values; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.rpc.RequestHandlerProvider; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import javax.security.auth.x500.X500Principal; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.math.BigInteger; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; + +import static com.yahoo.vespa.config.server.rpc.security.MultiTenantRpcAuthorizer.Mode.ENFORCE; +import static java.time.temporal.ChronoUnit.DAYS; +import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author bjorncs + */ +public class MultiTenantRpcAuthorizerTest { + + private static final List<X509Certificate> PEER_CERTIFICATE_CHAIN = List.of(createDummyCertificate()); + private static final ApplicationId APPLICATION_ID = ApplicationId.from("mytenant", "myapplication", "default"); + private static final ApplicationId EVIL_APP_ID = ApplicationId.from("malice", "malice-app", "default"); + private static final HostName HOSTNAME = HostName.from("myhostname"); + private static final FileReference FILE_REFERENCE = new FileReference("myfilereference"); + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void configserver_can_access_files_and_config() throws InterruptedException, ExecutionException { + RpcAuthorizer authorizer = createAuthorizer(new NodeIdentity.Builder(NodeType.config).build(), + new HostRegistry<>()); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + authorizer.authorizeConfigRequest(configRequest) + .get(); + + Request fileRequest = createFileRequest(FILE_REFERENCE); + authorizer.authorizeFileRequest(fileRequest) + .get(); + } + + @Test + public void tenant_node_can_access_its_own_files_and_config() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(APPLICATION_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + hostRegistry.update(APPLICATION_ID.tenant(), List.of(HOSTNAME.value())); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + authorizer.authorizeConfigRequest(configRequest) + .get(); + + Request fileRequest = createFileRequest(FILE_REFERENCE); + authorizer.authorizeFileRequest(fileRequest) + .get(); + } + + @Test + public void proxy_node_can_access_lbservice_config() throws ExecutionException, InterruptedException { + RpcAuthorizer authorizer = createAuthorizer(new NodeIdentity.Builder(NodeType.proxy).build(), new HostRegistry<>()); + + Request configRequest = createConfigRequest( + new ConfigKey<>(LbServicesConfig.CONFIG_DEF_NAME, "*", LbServicesConfig.CONFIG_DEF_NAMESPACE), + HOSTNAME); + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_node_can_access_routing_config() throws ExecutionException, InterruptedException { + RpcAuthorizer authorizer = createAuthorizer(new NodeIdentity.Builder(NodeType.tenant).build(), new HostRegistry<>()); + + Request configRequest = createConfigRequest( + new ConfigKey<>(RoutingConfig.CONFIG_DEF_NAME, "*", RoutingConfig.CONFIG_DEF_NAMESPACE), + HOSTNAME); + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_node_cannot_access_lbservice_config() throws ExecutionException, InterruptedException { + RpcAuthorizer authorizer = createAuthorizer(new NodeIdentity.Builder(NodeType.tenant).build(), new HostRegistry<>()); + + Request configRequest = createConfigRequest( + new ConfigKey<>(LbServicesConfig.CONFIG_DEF_NAME, "*", LbServicesConfig.CONFIG_DEF_NAMESPACE), + HOSTNAME); + + exceptionRule.expectMessage("Node with type 'tenant' is not allowed to access global config [name=lb-services,namespace=cloud.config,configId=*]"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_node_cannot_access_other_files() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(APPLICATION_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + hostRegistry.update(APPLICATION_ID.tenant(), List.of(HOSTNAME.value())); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request fileRequest = createFileRequest(new FileReference("other-file-reference")); + + exceptionRule.expectMessage("Peer is not allowed to access file other-file-reference"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeFileRequest(fileRequest) + .get(); + } + + @Test + public void tenant_node_cannot_access_other_config() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(EVIL_APP_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + hostRegistry.update(APPLICATION_ID.tenant(), List.of(HOSTNAME.value())); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + + exceptionRule.expectMessage("Peer is not allowed to access config for owned by mytenant.myapplication. Peer is owned by malice.malice-app"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_node_must_be_registered_in_host_registry() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(EVIL_APP_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + + exceptionRule.expectMessage("Host 'myhostname' not found in host registry"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_must_have_a_request_handler() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(EVIL_APP_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + hostRegistry.update(EVIL_APP_ID.tenant(), List.of(HOSTNAME.value())); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>("name", "configid", "namespace"), HOSTNAME); + + exceptionRule.expectMessage("No handler exists for tenant 'malice'"); + exceptionRule.expectCause(instanceOf(AuthorizationException.class)); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + @Test + public void tenant_node_not_in_hostregistry_allowed_to_access_sentinel_config() throws ExecutionException, InterruptedException { + NodeIdentity identity = new NodeIdentity.Builder(NodeType.tenant) + .applicationId(APPLICATION_ID) + .build(); + + HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); + + RpcAuthorizer authorizer = createAuthorizer(identity, hostRegistry); + + Request configRequest = createConfigRequest(new ConfigKey<>(SentinelConfig.CONFIG_DEF_NAME, "configid", SentinelConfig.CONFIG_DEF_NAMESPACE), HOSTNAME); + + authorizer.authorizeConfigRequest(configRequest) + .get(); + } + + + private static RpcAuthorizer createAuthorizer(NodeIdentity identity, HostRegistry<TenantName> hostRegistry) { + return new MultiTenantRpcAuthorizer( + new StaticNodeIdentifier(identity), + hostRegistry, + createRequestHandlerProviderMock(), + new DirectExecutor(), + ENFORCE); + } + + private static Request createConfigRequest(ConfigKey<?> configKey, HostName hostName) { + return mockJrtRpcRequest(createConfigPayload(configKey, hostName.value())); + } + + private static Request createFileRequest(FileReference fileReference) { + return mockJrtRpcRequest(fileReference.value()); + } + + private static RequestHandlerProvider createRequestHandlerProviderMock() { + RequestHandler requestHandler = mock(RequestHandler.class); + when(requestHandler.hasApplication(APPLICATION_ID, Optional.empty())).thenReturn(true); + when(requestHandler.resolveApplicationId(HOSTNAME.value())).thenReturn(APPLICATION_ID); + when(requestHandler.listFileReferences(APPLICATION_ID)).thenReturn(Set.of(FILE_REFERENCE)); + + RequestHandlerProvider handlerProvider = mock(RequestHandlerProvider.class); + when(handlerProvider.getRequestHandler(APPLICATION_ID.tenant())).thenReturn(Optional.of(requestHandler)); + when(handlerProvider.getRequestHandler(EVIL_APP_ID.tenant())).thenReturn(Optional.empty()); + return handlerProvider; + } + + private static Request mockJrtRpcRequest(String payload) { + SecurityContext securityContext = mock(SecurityContext.class); + when(securityContext.peerCertificateChain()).thenReturn(PEER_CERTIFICATE_CHAIN); + Target target = mock(Target.class); + when(target.getSecurityContext()).thenReturn(Optional.of(securityContext)); + Request request = mock(Request.class); + when(request.target()).thenReturn(target); + Values values = new Values(); + values.add(new StringValue(payload)); + when(request.parameters()).thenReturn(values); + return request; + } + + private static String createConfigPayload(ConfigKey<?> configKey, String hostname) { + Slime data = new Slime(); + Cursor request = data.setObject(); + request.setString("defName", configKey.getName()); + request.setString("defNamespace", configKey.getNamespace()); + request.setString("defMD5", configKey.getMd5()); + request.setString("configId", configKey.getConfigId()); + request.setString("clientHostname", hostname); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + new JsonFormat(false).encode(out, data); + return new String(out.toByteArray()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static X509Certificate createDummyCertificate() { + return X509CertificateBuilder.fromKeypair( + KeyUtils.generateKeypair(KeyAlgorithm.EC), + new X500Principal("CN=" + HOSTNAME), + Instant.EPOCH, + Instant.EPOCH.plus(1, DAYS), + SignatureAlgorithm.SHA256_WITH_ECDSA, + BigInteger.ONE) + .build(); + } + + private static class DirectExecutor implements Executor { + + @Override + public void execute(Runnable command) { + command.run(); + } + } + + private static class StaticNodeIdentifier implements NodeIdentifier { + final NodeIdentity identity; + + StaticNodeIdentifier(NodeIdentity identity) { + this.identity = identity; + } + + @Override + public NodeIdentity identifyNode(List<X509Certificate> peerCertificateChain) throws NodeIdentifierException { + return identity; + } + } + +}
\ No newline at end of file diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 3626c6269cc..d3f364e30ac 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -46,7 +46,6 @@ public class LocalSessionRepoTest { } private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { - GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder().curator(new MockCurator()).build(); TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(temporaryFolder.newFolder(), tenantName); if (createInitialSessions) { IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "1")); @@ -54,11 +53,14 @@ public class LocalSessionRepoTest { IOUtils.copyDirectory(testApp, new File(tenantFileSystemDirs.sessionsPath(), "3")); } clock = new ManualClock(Instant.ofEpochSecond(1)); + GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder().curator(new MockCurator()) + .clock(clock) + .build(); LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, - TenantApplications.create(new MockCurator(), new MockReloadHandler(), tenantName), + TenantApplications.create(globalComponentRegistry, new MockReloadHandler(), tenantName), tenantFileSystemDirs, new HostRegistry<>(), tenantName); - repo = new LocalSessionRepo(tenantFileSystemDirs, loader, clock, 5, globalComponentRegistry.getCurator()); + repo = new LocalSessionRepo(tenantName, globalComponentRegistry, tenantFileSystemDirs, loader); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index a4432dcbfcd..96caff9b3a7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -16,7 +16,7 @@ import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockReloadHandler; -import com.yahoo.vespa.config.server.SuperModelGenerationCounter; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; @@ -54,13 +54,11 @@ public class LocalSessionTest { private Curator curator; private ConfigCurator configCurator; private TenantFileSystemDirs tenantFileSystemDirs; - private SuperModelGenerationCounter superModelGenerationCounter; @Before public void setupTest() { curator = new MockCurator(); configCurator = ConfigCurator.create(curator); - superModelGenerationCounter = new SuperModelGenerationCounter(curator); tenantFileSystemDirs = new TenantFileSystemDirs(Files.createTempDir(), TenantName.from("test_tenant")); } @@ -130,11 +128,9 @@ public class LocalSessionTest { String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(3)).getAbsolute(); assertTrue(configCurator.exists(sessionNode)); assertTrue(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - long gen = superModelGenerationCounter.get(); NestedTransaction transaction = new NestedTransaction(); session.delete(transaction); transaction.commit(); - assertThat(superModelGenerationCounter.get(), is(gen + 1)); assertFalse(configCurator.exists(sessionNode)); assertFalse(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); } @@ -196,7 +192,7 @@ public class LocalSessionTest { zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); sessionDir.createNewFile(); - TenantApplications applications = TenantApplications.create(curator, new MockReloadHandler(), tenant); + TenantApplications applications = TenantApplications.create(new TestComponentRegistry.Builder().curator(curator).build(), new MockReloadHandler(), tenant); applications.createApplication(zkc.readApplicationId()); return new LocalSession(tenant, sessionId, preparer, new SessionContext( @@ -205,7 +201,6 @@ public class LocalSessionTest { sessionDir, applications, new HostRegistry<>(), - superModelGenerationCounter, flagSource)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index f2fb4aa1c40..f5fd6053b07 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -6,10 +6,13 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; - +import com.yahoo.config.model.api.ContainerEndpoint; import org.junit.Test; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -17,6 +20,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -26,6 +30,15 @@ import static org.junit.Assert.assertTrue; */ public class PrepareParamsTest { + private static final String rotation = "rotation-042.vespa.a02.yahoodns.net"; + private static final String vespaVersion = "6.37.49"; + private static final String request = "http://foo:19071/application/v2/tenant/foo/application/bar?" + + PrepareParams.DRY_RUN_PARAM_NAME + "=true&" + + PrepareParams.VERBOSE_PARAM_NAME+ "=true&" + + PrepareParams.IGNORE_VALIDATION_PARAM_NAME + "=false&" + + PrepareParams.APPLICATION_NAME_PARAM_NAME + "=baz&" + + PrepareParams.VESPA_VERSION_PARAM_NAME + "=" + vespaVersion; + @Test public void testCorrectParsing() { PrepareParams prepareParams = createParams("http://foo:19071/application/v2/", TenantName.defaultName()); @@ -38,15 +51,6 @@ public class PrepareParamsTest { assertTrue(prepareParams.getTimeoutBudget().hasTimeLeft()); assertThat(prepareParams.rotations().size(), is(0)); } - - private static final String rotation = "rotation-042.vespa.a02.yahoodns.net"; - private static final String vespaVersion = "6.37.49"; - private static final String request = "http://foo:19071/application/v2/tenant/foo/application/bar?" + - PrepareParams.DRY_RUN_PARAM_NAME + "=true&" + - PrepareParams.VERBOSE_PARAM_NAME+ "=true&" + - PrepareParams.IGNORE_VALIDATION_PARAM_NAME + "=false&" + - PrepareParams.APPLICATION_NAME_PARAM_NAME + "=baz&" + - PrepareParams.VESPA_VERSION_PARAM_NAME + "=" + vespaVersion; @Test public void testCorrectParsingWithRotation() { @@ -77,6 +81,31 @@ public class PrepareParamsTest { assertThat(rotations, containsInAnyOrder(new Rotation(rotation), new Rotation(rotationTwo))); } + @Test + public void testCorrectParsingWithContainerEndpoints() { + var endpoints = List.of(new ContainerEndpoint("qrs1", + List.of("c1.example.com", + "c2.example.com")), + new ContainerEndpoint("qrs2", + List.of("c3.example.com", + "c4.example.com"))); + var param = "[\n" + + " {\n" + + " \"clusterId\": \"qrs1\",\n" + + " \"names\": [\"c1.example.com\", \"c2.example.com\"]\n" + + " },\n" + + " {\n" + + " \"clusterId\": \"qrs2\",\n" + + " \"names\": [\"c3.example.com\", \"c4.example.com\"]\n" + + " }\n" + + "]"; + + var encoded = URLEncoder.encode(param, StandardCharsets.UTF_8); + var prepareParams = createParams(request + "&" + PrepareParams.CONTAINER_ENDPOINTS_PARAM_NAME + + "=" + encoded, TenantName.from("foo")); + assertEquals(endpoints, prepareParams.containerEndpoints()); + } + // Create PrepareParams from a request (based on uri and tenant name) private static PrepareParams createParams(String uri, TenantName tenantName) { return PrepareParams.fromHttpRequest( @@ -84,4 +113,5 @@ public class PrepareParamsTest { tenantName, Duration.ofSeconds(60)); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java index 9dda653dbc1..83183a27666 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; @@ -100,11 +101,12 @@ public class RemoteSessionRepoTest { public void testBadApplicationRepoOnActivate() { long sessionId = 3L; TenantName mytenant = TenantName.from("mytenant"); - TenantApplications applicationRepo = TenantApplications.create(curator, new MockReloadHandler(), mytenant); + GlobalComponentRegistry registry = new TestComponentRegistry.Builder().curator(curator).build(); + TenantApplications applicationRepo = TenantApplications.create(registry, new MockReloadHandler(), mytenant); curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data - Tenant tenant = TenantBuilder.create(new TestComponentRegistry.Builder().curator(curator).build(), mytenant) - .withApplicationRepo(applicationRepo) - .build(); + Tenant tenant = TenantBuilder.create(registry, mytenant) + .withApplicationRepo(applicationRepo) + .build(); curator.create(TenantRepository.getSessionsPath(mytenant)); remoteSessionRepo = tenant.getRemoteSessionRepo(); assertThat(remoteSessionRepo.listSessions().size(), is(0)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 9ad90e84d86..651dde375ee 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -1,22 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.CertificateNotReadyException; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.MockReloadHandler; -import com.yahoo.vespa.config.server.SuperModelGenerationCounter; +import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudgetTest; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; @@ -27,9 +29,11 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.Rotations; +import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; - import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Before; @@ -42,6 +46,7 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -49,8 +54,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -70,7 +75,7 @@ public class SessionPreparerTest { private SessionPreparer preparer; private TestComponentRegistry componentRegistry; private MockFileDistributionFactory fileDistributionFactory; - + private MockSecretStore secretStore = new MockSecretStore(); @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -105,7 +110,8 @@ public class SessionPreparerTest { componentRegistry.getStaticConfigDefinitionRepo(), curator, componentRegistry.getZone(), - flagSource); + flagSource, + secretStore); } @Test(expected = InvalidApplicationException.class) @@ -170,6 +176,10 @@ public class SessionPreparerTest { assertThat(zkc.readApplicationId(), is(origId)); } + private List<ContainerEndpoint> readContainerEndpoints(ApplicationId application) { + return new ContainerEndpointsCache(tenantPath, curator).read(application); + } + private Set<Rotation> readRotationsFromZK(ApplicationId applicationId) { return new Rotations(curator, tenantPath).readRotationsFromZooKeeper(applicationId); } @@ -205,6 +215,85 @@ public class SessionPreparerTest { assertThat(readRotationsFromZK(applicationId), contains(new Rotation(rotations))); } + @Test + public void require_that_rotations_are_written_as_container_endpoints() throws Exception { + var rotations = "app1.tenant1.global.vespa.example.com,rotation-042.vespa.global.routing"; + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).rotations(rotations).build(); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + var expected = List.of(new ContainerEndpoint("qrs", + List.of("app1.tenant1.global.vespa.example.com", + "rotation-042.vespa.global.routing"))); + assertEquals(expected, readContainerEndpoints(applicationId)); + } + + @Test + public void require_that_container_endpoints_are_written() throws Exception { + var endpoints = "[\n" + + " {\n" + + " \"clusterId\": \"foo\",\n" + + " \"names\": [\n" + + " \"foo.app1.tenant1.global.vespa.example.com\",\n" + + " \"rotation-042.vespa.global.routing\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"clusterId\": \"bar\",\n" + + " \"names\": [\n" + + " \"bar.app1.tenant1.global.vespa.example.com\",\n" + + " \"rotation-043.vespa.global.routing\"\n" + + " ]\n" + + " }\n" + + "]"; + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId) + .containerEndpoints(endpoints) + .build(); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + var expected = List.of(new ContainerEndpoint("foo", + List.of("foo.app1.tenant1.global.vespa.example.com", + "rotation-042.vespa.global.routing")), + new ContainerEndpoint("bar", + List.of("bar.app1.tenant1.global.vespa.example.com", + "rotation-043.vespa.global.routing"))); + assertEquals(expected, readContainerEndpoints(applicationId)); + } + + @Test + public void require_that_tlssecretkey_is_written() throws IOException { + var tlskey = "vespa.tlskeys.tenant1--app1"; + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); + secretStore.put(tlskey+"-cert", "CERT"); + secretStore.put(tlskey+"-key", "KEY"); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + // Read from zk and verify cert and key are available + Optional<TlsSecrets> tlsSecrets = new TlsSecretsKeys(curator, tenantPath, secretStore).readTlsSecretsKeyFromZookeeper(applicationId); + assertTrue(tlsSecrets.isPresent()); + assertEquals("KEY", tlsSecrets.get().key()); + assertEquals("CERT", tlsSecrets.get().certificate()); + } + + @Test(expected = CertificateNotReadyException.class) + public void require_that_tlssecretkey_is_missing_when_not_in_secretstore() throws IOException { + var tlskey = "vespa.tlskeys.tenant1--app1"; + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + } + + @Test(expected = CertificateNotReadyException.class) + public void require_that_tlssecretkey_is_missing_when_certificate_not_in_secretstore() throws IOException { + var tlskey = "vespa.tlskeys.tenant1--app1"; + var applicationId = applicationId("test"); + var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build(); + secretStore.put(tlskey+"-key", "KEY"); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + } + private void prepare(File app) throws IOException { prepare(app, new PrepareParams.Builder().build()); } @@ -217,9 +306,8 @@ public class SessionPreparerTest { return new SessionContext(app, new SessionZooKeeperClient(curator, sessionsPath), app.getAppDir(), - TenantApplications.create(curator, new MockReloadHandler(), TenantName.from("tenant")), + TenantApplications.create(componentRegistry, new MockReloadHandler(), TenantName.from("tenant")), new HostRegistry<>(), - new SuperModelGenerationCounter(curator), flagSource); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java index 95f6c7718e2..b2ad0af8f9a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionTest.java @@ -21,7 +21,7 @@ public class SessionTest { public boolean isPrepared = false; public MockSessionPreparer() { - super(null, null, null, null, null, null, new MockCurator(), null, null); + super(null, null, null, null, null, null, new MockCurator(), null, null, null); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java new file mode 100644 index 00000000000..053a3f7a15d --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -0,0 +1,50 @@ +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.slime.Slime; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author ogronnesby + */ +public class ContainerEndpointSerializerTest { + + @Test + public void readSingleEndpoint() { + final var slime = new Slime(); + final var entry = slime.setObject(); + + entry.setString("clusterId", "foobar"); + final var entryNames = entry.setArray("names"); + entryNames.addString("a"); + entryNames.addString("b"); + + final var endpoint = ContainerEndpointSerializer.endpointFromSlime(slime.get()); + assertEquals("foobar", endpoint.clusterId().toString()); + assertEquals(List.of("a", "b"), endpoint.names()); + } + + @Test + public void writeReadSingleEndpoint() { + final var endpoint = new ContainerEndpoint("foo", List.of("a", "b")); + final var serialized = new Slime(); + ContainerEndpointSerializer.endpointToSlime(serialized.setObject(), endpoint); + final var deserialized = ContainerEndpointSerializer.endpointFromSlime(serialized.get()); + + assertEquals(endpoint, deserialized); + } + + @Test + public void writeReadEndpoints() { + final var endpoints = List.of(new ContainerEndpoint("foo", List.of("a", "b"))); + final var serialized = ContainerEndpointSerializer.endpointListToSlime(endpoints); + final var deserialized = ContainerEndpointSerializer.endpointListFromSlime(serialized); + + assertEquals(endpoints, deserialized); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java new file mode 100644 index 00000000000..4400b424d1b --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java @@ -0,0 +1,36 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ContainerEndpointsCacheTest { + @Test + public void readWriteFromCache() { + final var cache = new ContainerEndpointsCache(Path.createRoot(), new MockCurator()); + final var endpoints = List.of( + new ContainerEndpoint("the-cluster-1", List.of("a", "b", "c")) + ); + + cache.write(ApplicationId.defaultId(), endpoints); + + final var deserialized = cache.read(ApplicationId.defaultId()); + + assertEquals(endpoints, deserialized); + } + + @Test + public void readingNonExistingEntry() { + final var cache = new ContainerEndpointsCache(Path.createRoot(), new MockCurator()); + final var endpoints = cache.read(ApplicationId.defaultId()); + assertTrue(endpoints.isEmpty()); + } +}
\ No newline at end of file diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 6aa5aa7cd70..36bb7a926b5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -93,11 +93,11 @@ public class TenantRequestHandlerTest { Metrics sh = Metrics.createTestMetrics(); List<ReloadListener> listeners = new ArrayList<>(); listeners.add(listener); - server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), new HostRegistries(), curator); componentRegistry = new TestComponentRegistry.Builder() .curator(curator) .modelFactoryRegistry(createRegistry()) .build(); + server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), componentRegistry); } private void feedApp(File appDir, long sessionId, ApplicationId appId, boolean internalRedeploy) throws IOException { @@ -357,12 +357,12 @@ public class TenantRequestHandlerTest { configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); System.out.println(configNames); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "jdisc", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "jdisc", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "jdisc", "container.jdisc.config"))); - assertTrue(configNames.contains(new ConfigKey<>("specific", "jdisc", "project"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); + assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java index baab250a508..e140dae3650 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java @@ -38,7 +38,7 @@ public class TenantTest { TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); TenantName tenantName = TenantName.from(name); TenantBuilder tenantBuilder = TenantBuilder.create(componentRegistry, tenantName) - .withApplicationRepo(TenantApplications.create(new MockCurator(), new MockReloadHandler(), tenantName)); + .withApplicationRepo(TenantApplications.create(componentRegistry, new MockReloadHandler(), tenantName)); tenantRepository.addTenant(tenantBuilder); return tenantRepository.getTenant(tenantName); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index e8e2dd07756..8b8be1a27d7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -1,27 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertThat; - -import java.io.File; -import java.io.IOException; -import java.io.Reader; -import java.util.Collections; -import java.util.Optional; -import java.util.regex.Pattern; - +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.mock.MockCurator; @@ -30,7 +18,19 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.yahoo.io.IOUtils; +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.util.Collections; +import java.util.Optional; +import java.util.regex.Pattern; + +import static com.yahoo.config.provision.serialization.AllocatedHostsSerializer.toJson; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class ZKApplicationPackageTest { @@ -77,7 +77,7 @@ public class ZKApplicationPackageTest { assertFalse(zkApp.getFileRegistries().containsKey(new Version(0, 0, 0))); assertThat(zkApp.getFileRegistries().get(goodVersion).fileSourceHost(), is("dummyfiles")); AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); - assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(ALLOCATED_HOSTS.toJson()))); + assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertTrue(zkApp.getDeployment().isPresent()); @@ -90,7 +90,7 @@ public class ZKApplicationPackageTest { String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); - zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, ALLOCATED_HOSTS.toJson()); + zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS)); } private static class MockNodeFlavors extends NodeFlavors{ diff --git a/configserver/src/test/resources/deploy/advancedapp/services.xml b/configserver/src/test/resources/deploy/advancedapp/services.xml index e9b301bea52..e3d5aea585b 100644 --- a/configserver/src/test/resources/deploy/advancedapp/services.xml +++ b/configserver/src/test/resources/deploy/advancedapp/services.xml @@ -10,12 +10,12 @@ </slobroks> </admin> - <jdisc version="1.0"> + <container version="1.0"> <search /> <nodes> <node hostalias="node1" baseport='8000'/> </nodes> - </jdisc> + </container> <content version="1.0"> <redundancy>1</redundancy> diff --git a/configserver/src/test/resources/deploy/hosted-app/deployment.xml b/configserver/src/test/resources/deploy/hosted-app/deployment.xml new file mode 100644 index 00000000000..a92404c161d --- /dev/null +++ b/configserver/src/test/resources/deploy/hosted-app/deployment.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<deployment version='1.0'> + <prod global-service-id="qrs"> + <region active="true">us-north-1</region> + <region active="true">us-north-2</region> + </prod> +</deployment> diff --git a/configserver/src/test/resources/deploy/hosted-app/services.xml b/configserver/src/test/resources/deploy/hosted-app/services.xml new file mode 100644 index 00000000000..57bee6ce9c9 --- /dev/null +++ b/configserver/src/test/resources/deploy/hosted-app/services.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + <container id="qrs" version="1.0"> + <nodes count="2"/> + </container> +</services> |