diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-06-04 21:15:07 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2020-06-04 21:15:07 +0200 |
commit | be78d3ca58195e5461986f50e0597cfbf16f8810 (patch) | |
tree | 647530730d80028e81280b7dc25ab2bfbd13af9e /configserver | |
parent | 2021ee5263ac84b2f81ef748308ddf13b612b162 (diff) |
Merge TenantApplications and TenantRequestHandler
Move tests to ApplicationRepositoryTest or TenantApplicationsTest
Diffstat (limited to 'configserver')
10 files changed, 554 insertions, 735 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 4f04724a0a8..f7b9fff181b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -1,16 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.component.Version; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.host.HostValidator; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; @@ -19,15 +29,20 @@ import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import java.time.Clock; import java.time.Duration; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.util.stream.Collectors.toSet; + /** * The applications of a tenant, backed by ZooKeeper. * @@ -38,7 +53,7 @@ import java.util.stream.Collectors; * @author Ulf Lilleengen * @author jonmv */ -public class TenantApplications { +public class TenantApplications implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { private static final Logger log = Logger.getLogger(TenantApplications.class.getName()); @@ -46,24 +61,35 @@ public class TenantApplications { private final Path applicationsPath; private final Path locksPath; private final Curator.DirectoryCache directoryCache; - private final ReloadHandler reloadHandler; private final Executor zkWatcherExecutor; + private final Metrics metrics; + private final TenantName tenant; + private final List<ReloadListener> reloadListeners; + private final ConfigResponseFactory responseFactory; + private final HostRegistry<ApplicationId> hostRegistry; + private final ApplicationMapper applicationMapper = new ApplicationMapper(); + private final MetricUpdater tenantMetricUpdater; + private final Clock clock = Clock.systemUTC(); - private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant, - ExecutorService zkCacheExecutor, StripedExecutor<TenantName> zkWatcherExecutor) { - this.curator = curator; + + private TenantApplications(TenantName tenant, GlobalComponentRegistry registry) { + this.curator = registry.getCurator(); this.applicationsPath = TenantRepository.getApplicationsPath(tenant); this.locksPath = TenantRepository.getLocksPath(tenant); - this.reloadHandler = reloadHandler; - this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); - this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor); + this.tenant = tenant; + this.zkWatcherExecutor = command -> registry.getZkWatcherExecutor().execute(tenant, command); + this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, registry.getZkCacheExecutor()); this.directoryCache.start(); this.directoryCache.addListener(this::childEvent); + this.metrics = registry.getMetrics(); + this.reloadListeners = List.of(registry.getReloadListener()); + this.responseFactory = ConfigResponseFactory.create(registry.getConfigserverConfig()); + this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); + this.hostRegistry = registry.getHostRegistries().createApplicationHostRegistry(tenant); } - public static TenantApplications create(GlobalComponentRegistry registry, ReloadHandler reloadHandler, TenantName tenant) { - return new TenantApplications(registry.getCurator(), reloadHandler, tenant, - registry.getZkCacheExecutor(), registry.getZkWatcherExecutor()); + public static TenantApplications create(GlobalComponentRegistry registry, TenantName tenant) { + return new TenantApplications(tenant, registry); } /** @@ -132,7 +158,7 @@ public class TenantApplications { * Removes all applications not known to this from the config server state. */ public void removeUnusedApplications() { - reloadHandler.removeApplicationsExcept(Set.copyOf(activeApplications())); + removeApplicationsExcept(Set.copyOf(activeApplications())); } /** @@ -170,7 +196,7 @@ public class TenantApplications { } private void applicationRemoved(ApplicationId applicationId) { - reloadHandler.removeApplication(applicationId); + removeApplication(applicationId); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); } @@ -186,4 +212,202 @@ public class TenantApplications { return locksPath.append(id.serializedForm()); } + + /** + * Gets a config for the given app, or null if not found + */ + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); + } + return application.resolveConfig(req, responseFactory); + } + + // For testing only + long getApplicationGeneration(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.getApplicationGeneration(); + } + + private void notifyReloadListeners(ApplicationSet applicationSet) { + for (ReloadListener reloadListener : reloadListeners) { + reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + reloadListener.configActivated(applicationSet); + } + } + + /** + * Activates the config of the given app. Notifies listeners + * + * @param applicationSet the {@link ApplicationSet} to be reloaded + */ + @Override + public void reloadConfig(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + try (Lock lock = lock(id)) { + if ( ! exists(id)) + return; // Application was deleted before activation. + if (applicationSet.getApplicationGeneration() != requireActiveSessionOf(id)) + return; // Application activated a new session before we got here. + + setLiveApp(applicationSet); + notifyReloadListeners(applicationSet); + } + } + + @Override + public void removeApplication(ApplicationId applicationId) { + try (Lock lock = lock(applicationId)) { + if (exists(applicationId)) + return; // Application was deployed again. + + if (applicationMapper.hasApplication(applicationId, clock.instant())) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + } + } + } + + @Override + public void removeApplicationsExcept(Set<ApplicationId> applications) { + for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { + if ( ! applications.contains(activeApplication)) { + log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); + removeApplication(activeApplication); + } + } + } + + private void reloadListenersOnRemove(ApplicationId applicationId) { + for (ReloadListener listener : reloadListeners) { + listener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + listener.applicationRemoved(applicationId); + } + } + + private void setLiveApp(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + System.out.println("Setting live " + id); + Collection<String> hostsForApp = applicationSet.getAllHosts(); + hostRegistry.update(id, hostsForApp); + applicationSet.updateHostMetrics(); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + applicationMapper.register(id, applicationSet); + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, keyToMatch, recursive); + } + + private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> key : application.allConfigsProduced()) { + String configId = key.getConfigId(); + if (recursive) { + key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); + } else { + // Include first part of id as id + key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); + } + if (keyToMatch != null) { + String n = key.getName(); // Never null + String ns = key.getNamespace(); // Never null + if (n.equals(keyToMatch.getName()) && + ns.equals(keyToMatch.getNamespace()) && + configId.startsWith(keyToMatch.getConfigId()) && + !(configId.equals(keyToMatch.getConfigId()))) { + + if (!recursive) { + // For non-recursive, include the id segment we were searching for, and first part of the rest + key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); + } + ret.add(key); + } + } else { + ret.add(key); + } + } + return ret; + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, null, recursive); + } + + /** + * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers + * @return id segment with one extra level from the id appended + */ + String appendOneLevelOfId(String baseIdSegment, String id) { + if ("".equals(baseIdSegment)) return id.split("/")[0]; + String theRest = id.substring(baseIdSegment.length()); + if ("".equals(theRest)) return id; + theRest = theRest.replaceFirst("/", ""); + String theRestFirstSeg = theRest.split("/")[0]; + return baseIdSegment+"/"+theRestFirstSeg; + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigsProduced(); + } + + private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { + try { + return applicationMapper.getForVersion(appId, vespaVersion, clock.instant()); + } catch (VersionDoesNotExistException ex) { + throw new NotFoundException(String.format("%sNo such application (id %s): %s", TenantRepository.logPre(tenant), appId, ex.getMessage())); + } + } + + @Override + public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigIds(); + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + return hasHandler(appId, vespaVersion); + } + + private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { + return applicationMapper.hasApplicationForVersion(appId, vespaVersion, clock.instant()); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); + if (applicationId == null) { + applicationId = ApplicationId.defaultId(); + } + return applicationId; + } + + @Override + public Set<FileReference> listFileReferences(ApplicationId applicationId) { + return applicationMapper.listApplications(applicationId).stream() + .flatMap(app -> app.getModel().fileReferences().stream()) + .collect(toSet()); + } + + @Override + public void verifyHosts(ApplicationId key, Collection<String> newHosts) { + hostRegistry.verifyHosts(key, newHosts); + for (ReloadListener reloadListener : reloadListeners) { + reloadListener.verifyHostsAreAvailable(tenant, newHosts); + } + } + + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 43b25826507..e2776dc382e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -12,9 +12,7 @@ import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.SessionFactory; @@ -75,7 +73,7 @@ public class TenantRepository { private static final Logger log = Logger.getLogger(TenantRepository.class.getName()); private final Map<TenantName, Tenant> tenants = Collections.synchronizedMap(new LinkedHashMap<>()); - private final GlobalComponentRegistry globalComponentRegistry; + private final GlobalComponentRegistry componentRegistry; private final List<TenantListener> tenantListeners = Collections.synchronizedList(new ArrayList<>()); private final Curator curator; @@ -90,30 +88,30 @@ public class TenantRepository { /** * Creates a new tenant repository * - * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} */ @Inject - public TenantRepository(GlobalComponentRegistry globalComponentRegistry) { - this(globalComponentRegistry, true); + public TenantRepository(GlobalComponentRegistry componentRegistry) { + this(componentRegistry, true); } /** * Creates a new tenant repository * - * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} * @param useZooKeeperWatchForTenantChanges set to false for tests where you want to control adding and deleting * tenants yourself */ - public TenantRepository(GlobalComponentRegistry globalComponentRegistry, boolean useZooKeeperWatchForTenantChanges) { - this.globalComponentRegistry = globalComponentRegistry; - ConfigserverConfig configserverConfig = globalComponentRegistry.getConfigserverConfig(); + public TenantRepository(GlobalComponentRegistry componentRegistry, boolean useZooKeeperWatchForTenantChanges) { + this.componentRegistry = componentRegistry; + ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); this.throwExceptionIfBootstrappingFails = configserverConfig.throwIfBootstrappingTenantRepoFails(); - this.curator = globalComponentRegistry.getCurator(); - metricUpdater = globalComponentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); - this.tenantListeners.add(globalComponentRegistry.getTenantListener()); - this.zkCacheExecutor = globalComponentRegistry.getZkCacheExecutor(); - this.zkWatcherExecutor = globalComponentRegistry.getZkWatcherExecutor(); + this.curator = componentRegistry.getCurator(); + metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); + this.tenantListeners.add(componentRegistry.getTenantListener()); + this.zkCacheExecutor = componentRegistry.getZkCacheExecutor(); + this.zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); curator.framework().getConnectionStateListenable().addListener(this::stateChanged); curator.create(tenantsPath); @@ -210,34 +208,21 @@ public class TenantRepository { private void createTenant(TenantName tenantName, RequestHandler requestHandler, ReloadHandler reloadHandler) { if (tenants.containsKey(tenantName)) return; - TenantRequestHandler tenantRequestHandler = null; - if (requestHandler == null) { - tenantRequestHandler = new TenantRequestHandler(globalComponentRegistry.getMetrics(), - tenantName, - List.of(globalComponentRegistry.getReloadListener()), - ConfigResponseFactory.create(globalComponentRegistry.getConfigserverConfig()), - globalComponentRegistry); - requestHandler = tenantRequestHandler; - } - - if (reloadHandler == null && tenantRequestHandler != null) - reloadHandler = tenantRequestHandler; - - HostValidator<ApplicationId> hostValidator = tenantRequestHandler; - TenantApplications applicationRepo = TenantApplications.create(globalComponentRegistry, - reloadHandler, - tenantName); - - SessionFactory sessionFactory = new SessionFactory(globalComponentRegistry, applicationRepo, hostValidator, tenantName); - LocalSessionRepo localSessionRepo = new LocalSessionRepo(tenantName, globalComponentRegistry, sessionFactory); - RemoteSessionRepo remoteSessionRepo = new RemoteSessionRepo(globalComponentRegistry, + TenantApplications applicationRepo = TenantApplications.create(componentRegistry, tenantName); + if (requestHandler == null) + requestHandler = applicationRepo; + if (reloadHandler == null) + reloadHandler = applicationRepo; + SessionFactory sessionFactory = new SessionFactory(componentRegistry, applicationRepo, applicationRepo, tenantName); + LocalSessionRepo localSessionRepo = new LocalSessionRepo(tenantName, componentRegistry, sessionFactory); + RemoteSessionRepo remoteSessionRepo = new RemoteSessionRepo(componentRegistry, sessionFactory, reloadHandler, tenantName, applicationRepo); log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); - Tenant tenant = new Tenant(tenantName, sessionFactory, localSessionRepo, remoteSessionRepo, requestHandler, - reloadHandler, applicationRepo, globalComponentRegistry.getCurator()); + Tenant tenant = new Tenant(tenantName, sessionFactory, localSessionRepo, remoteSessionRepo, requestHandler, + reloadHandler, applicationRepo, componentRegistry.getCurator()); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java deleted file mode 100644 index 25d6f194fdc..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.component.Version; -import com.yahoo.config.FileReference; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.NotFoundException; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationMapper; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.application.VersionDoesNotExistException; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import com.yahoo.vespa.curator.Lock; - -import java.time.Clock; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; - -import static java.util.stream.Collectors.toSet; - -/** - * A per tenant request handler, for handling reload (activate application) and getConfig requests for - * a set of applications belonging to a tenant. - * - * @author Harald Musum - */ -public class TenantRequestHandler implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { - - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(TenantRequestHandler.class.getName()); - - private final Metrics metrics; - private final TenantName tenant; - private final List<ReloadListener> reloadListeners; - private final ConfigResponseFactory responseFactory; - private final HostRegistry<ApplicationId> hostRegistry; - private final ApplicationMapper applicationMapper = new ApplicationMapper(); - private final MetricUpdater tenantMetricUpdater; - private final Clock clock = Clock.systemUTC(); - private final TenantApplications applications; - - public TenantRequestHandler(Metrics metrics, - TenantName tenant, - List<ReloadListener> reloadListeners, - ConfigResponseFactory responseFactory, - GlobalComponentRegistry registry) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. - this.metrics = metrics; - this.tenant = tenant; - this.reloadListeners = List.copyOf(reloadListeners); - this.responseFactory = responseFactory; - this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); - this.hostRegistry = registry.getHostRegistries().createApplicationHostRegistry(tenant); - this.applications = TenantApplications.create(registry, this, tenant); - - } - - /** - * Gets a config for the given app, or null if not found - */ - @Override - public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); - } - return application.resolveConfig(req, responseFactory); - } - - // For testing only - long getApplicationGeneration(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.getApplicationGeneration(); - } - - private void notifyReloadListeners(ApplicationSet applicationSet) { - for (ReloadListener reloadListener : reloadListeners) { - reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - reloadListener.configActivated(applicationSet); - } - } - - /** - * Activates the config of the given app. Notifies listeners - * - * @param applicationSet the {@link ApplicationSet} to be reloaded - */ - @Override - public void reloadConfig(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); - try (Lock lock = applications.lock(id)) { - if ( ! applications.exists(id)) - return; // Application was deleted before activation. - if (applicationSet.getApplicationGeneration() != applications.requireActiveSessionOf(id)) - return; // Application activated a new session before we got here. - - setLiveApp(applicationSet); - notifyReloadListeners(applicationSet); - } - } - - @Override - public void removeApplication(ApplicationId applicationId) { - try (Lock lock = applications.lock(applicationId)) { - if (applications.exists(applicationId)) - return; // Application was deployed again. - - if (applicationMapper.hasApplication(applicationId, clock.instant())) { - applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); - reloadListenersOnRemove(applicationId); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); - } - } - } - - @Override - public void removeApplicationsExcept(Set<ApplicationId> applications) { - for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { - if ( ! applications.contains(activeApplication)) { - log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); - removeApplication(activeApplication); - } - } - } - - private void reloadListenersOnRemove(ApplicationId applicationId) { - for (ReloadListener listener : reloadListeners) { - listener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - listener.applicationRemoved(applicationId); - } - } - - private void setLiveApp(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); - Collection<String> hostsForApp = applicationSet.getAllHosts(); - hostRegistry.update(id, hostsForApp); - applicationSet.updateHostMetrics(); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - applicationMapper.register(id, applicationSet); - } - - @Override - public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { - Application application = getApplication(appId, vespaVersion); - return listConfigs(application, keyToMatch, recursive); - } - - private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { - Set<ConfigKey<?>> ret = new LinkedHashSet<>(); - for (ConfigKey<?> key : application.allConfigsProduced()) { - String configId = key.getConfigId(); - if (recursive) { - key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); - } else { - // Include first part of id as id - key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); - } - if (keyToMatch != null) { - String n = key.getName(); // Never null - String ns = key.getNamespace(); // Never null - if (n.equals(keyToMatch.getName()) && - ns.equals(keyToMatch.getNamespace()) && - configId.startsWith(keyToMatch.getConfigId()) && - !(configId.equals(keyToMatch.getConfigId()))) { - - if (!recursive) { - // For non-recursive, include the id segment we were searching for, and first part of the rest - key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); - } - ret.add(key); - } - } else { - ret.add(key); - } - } - return ret; - } - - @Override - public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { - Application application = getApplication(appId, vespaVersion); - return listConfigs(application, null, recursive); - } - - /** - * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers - * @return id segment with one extra level from the id appended - */ - String appendOneLevelOfId(String baseIdSegment, String id) { - if ("".equals(baseIdSegment)) return id.split("/")[0]; - String theRest = id.substring(baseIdSegment.length()); - if ("".equals(theRest)) return id; - theRest = theRest.replaceFirst("/", ""); - String theRestFirstSeg = theRest.split("/")[0]; - return baseIdSegment+"/"+theRestFirstSeg; - } - - @Override - public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.allConfigsProduced(); - } - - private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { - try { - return applicationMapper.getForVersion(appId, vespaVersion, clock.instant()); - } catch (VersionDoesNotExistException ex) { - throw new NotFoundException(String.format("%sNo such application (id %s): %s", TenantRepository.logPre(tenant), appId, ex.getMessage())); - } - } - - @Override - public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.allConfigIds(); - } - - @Override - public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { - return hasHandler(appId, vespaVersion); - } - - private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { - return applicationMapper.hasApplicationForVersion(appId, vespaVersion, clock.instant()); - } - - @Override - public ApplicationId resolveApplicationId(String hostName) { - ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); - if (applicationId == null) { - applicationId = ApplicationId.defaultId(); - } - return applicationId; - } - - @Override - public Set<FileReference> listFileReferences(ApplicationId applicationId) { - return applicationMapper.listApplications(applicationId).stream() - .flatMap(app -> app.getModel().fileReferences().stream()) - .collect(toSet()); - } - - @Override - public void verifyHosts(ApplicationId key, Collection<String> newHosts) { - hostRegistry.verifyHosts(key, newHosts); - for (ReloadListener reloadListener : reloadListeners) { - reloadListener.verifyHostsAreAvailable(tenant, newHosts); - } - } - - TenantApplications applications() { return applications; } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d44be294713..3e5b539a5bf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -2,7 +2,11 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.provision.AllocatedHosts; @@ -20,7 +24,14 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -36,6 +47,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModelFactory; +import org.hamcrest.core.Is; +import org.jetbrains.annotations.NotNull; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -76,6 +90,8 @@ public class ApplicationRepositoryTest { private final static File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only"); private final static File testAppJdiscOnlyRestart = new File("src/test/apps/app-jdisc-only-restart"); private final static File testAppLogServerWithContainer = new File("src/test/apps/app-logserver-with-container"); + private final static File app1 = new File("src/test/apps/cs1"); + private final static File app2 = new File("src/test/apps/cs2"); private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); @@ -94,12 +110,21 @@ public class ApplicationRepositoryTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { Curator curator = new MockCurator(); - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() - .curator(curator) - .build()); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) + .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .build()) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); @@ -529,6 +554,93 @@ public class ApplicationRepositoryTest { assertEquals(Session.Status.DEACTIVATE, firstSession.getStatus()); } + @Test + public void testResolveForAppId() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337 , config.intval()); + } + + @Test + public void testResolveConfigForMultipleApps() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + ApplicationId appId2 = new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("myapp2") + .instanceName("default") + .build(); + applicationRepository.deploy(app2, new PrepareParams.Builder() + .applicationId(appId2) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337, config.intval()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler2 = reloadConfig(appId2); + SimpletypesConfig config2 = resolve(SimpletypesConfig.class, requestHandler2, appId2, vespaVersion); + assertEquals(1330, config2.intval()); + + assertTrue(requestHandler.hasApplication(applicationId(), Optional.of(vespaVersion))); + assertThat(requestHandler.resolveApplicationId("doesnotexist"), Is.is(ApplicationId.defaultId())); + assertThat(requestHandler.resolveApplicationId("mytesthost"), Is.is(new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("testapp").build())); // Host set in application package. + } + + @Test + public void testResolveMultipleVersions() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337, config.intval()); + + // TODO: Revisit this test, I cannot see that we create a model for version 3.2.1 + config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), new Version(3, 2, 1)); + assertThat(config.intval(), Is.is(1337)); + } + + @Test + public void testResolveForDeletedApp() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337 , config.intval()); + + applicationRepository.delete(applicationId()); + + exceptionRule.expect(com.yahoo.vespa.config.server.NotFoundException.class); + exceptionRule.expectMessage(containsString("No such application id: test1.testapp")); + resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, @@ -569,7 +681,6 @@ public class ApplicationRepositoryTest { return applicationRepository.getMetadataFromLocalSession(tenant, sessionId); } - /** Stores all added or set values for each metric and context. */ static class MockMetric implements Metric { @@ -591,7 +702,6 @@ public class ApplicationRepositoryTest { return new Context(properties); } - private static class Context implements Metric.Context { private final Map<String, ?> point; @@ -604,4 +714,48 @@ public class ApplicationRepositoryTest { } + private <T extends ConfigInstance> T resolve(Class<T> clazz, + RequestHandler applications, + ApplicationId appId, + Version vespaVersion) { + String configId = ""; + ConfigResponse response = getConfigResponse(clazz, applications, appId, vespaVersion, configId); + return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); + } + + private <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, + RequestHandler applications, + ApplicationId appId, + Version vespaVersion, + String configId) { + return applications.resolveConfig(appId, new GetConfigRequest() { + @Override + public ConfigKey<T> getConfigKey() { + return new ConfigKey<>(clazz, configId); + } + + @Override + public DefContent getDefContent() { + return DefContent.fromClass(clazz); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return Optional.of(VespaVersion.fromString(vespaVersion.toFullString())); + } + + @Override + public boolean noCache() { + return false; + } + }, Optional.empty()); + } + + @NotNull + private RequestHandler reloadConfig(ApplicationId applicationId) { + RequestHandler requestHandler = tenantRepository.getTenant(applicationId.tenant()).getRequestHandler(); + ((TenantApplications) requestHandler).reloadConfig(applicationRepository.getActiveSession(applicationId).ensureApplicationLoaded()); + return requestHandler; + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index b3dca7c73f3..ec5648757f1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -20,7 +21,6 @@ import com.yahoo.vespa.config.server.session.MockFileDistributionFactory; import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.MockTenantListener; import com.yahoo.vespa.config.server.tenant.TenantListener; -import com.yahoo.vespa.config.server.tenant.TenantRequestHandlerTest; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -106,7 +106,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) .sessionLifetime(5)); private ConfigDefinitionRepo defRepo = new StaticConfigDefinitionRepo(); - private TenantRequestHandlerTest.MockReloadListener reloadListener = new TenantRequestHandlerTest.MockReloadListener(); + private ReloadListener reloadListener = new TenantApplicationsTest.MockReloadListener(); private MockTenantListener tenantListener = new MockTenantListener(); private Optional<PermanentApplicationPackage> permanentApplicationPackage = Optional.empty(); private HostRegistries hostRegistries = new HostRegistries(); @@ -156,6 +156,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder reloadListener(ReloadListener reloadListener) { + this.reloadListener = reloadListener; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 33932a678b7..969040174dd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -1,23 +1,49 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.server.MockReloadHandler; - +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.VespaModelFactory; import org.apache.curator.framework.CuratorFramework; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; - +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -25,14 +51,34 @@ import static org.junit.Assert.*; public class TenantApplicationsTest { private static final TenantName tenantName = TenantName.from("tenant"); + private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); - private Curator curator; + private MockReloadListener listener = new MockReloadListener(); private CuratorFramework curatorFramework; + private TestComponentRegistry componentRegistry; + private TenantApplications applications; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); @Before - public void setup() { - curator = new MockCurator(); + public void setup() throws IOException { + Curator curator = new MockCurator(); curatorFramework = curator.framework(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) + .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .build()) + .modelFactoryRegistry(createRegistry()) + .reloadListener(listener) + .build(); + TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); + tenantRepository.addTenant(tenantName); + applications = TenantApplications.create(componentRegistry, tenantName); } @Test @@ -94,29 +140,71 @@ public class TenantApplicationsTest { assertThat(repo.activeApplications().size(), is(0)); } - @Test - public void require_that_reload_handler_is_called_when_apps_are_removed() throws Exception { - ApplicationId foo = createApplicationId("foo"); - writeApplicationData(foo, 3L); - writeApplicationData(createApplicationId("bar"), 4L); - MockReloadHandler reloadHandler = new MockReloadHandler(); - TenantApplications repo = createZKAppRepo(reloadHandler); - assertNull(reloadHandler.lastRemoved); - repo.createDeleteTransaction(foo).commit(); - long endTime = System.currentTimeMillis() + 60_000; - while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { - Thread.sleep(100); + public static class MockReloadListener implements ReloadListener { + public AtomicInteger reloaded = new AtomicInteger(0); + AtomicInteger removed = new AtomicInteger(0); + Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); + + @Override + public void configActivated(ApplicationSet application) { + reloaded.incrementAndGet(); + } + + @Override + public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { + tenantHosts.put(tenant.value(), newHosts); + } + + @Override + public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { + } + + @Override + public void applicationRemoved(ApplicationId applicationId) { + removed.incrementAndGet(); } - assertNotNull(reloadHandler.lastRemoved); - assertThat(reloadHandler.lastRemoved.serializedForm(), is(foo.serializedForm())); } - private TenantApplications createZKAppRepo() { - return createZKAppRepo(new MockReloadHandler()); + private void assertdefaultAppNotFound() { + assertFalse(applications.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); + } + + @Test + public void testListConfigs() throws IOException, SAXException { + assertdefaultAppNotFound(); + + VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); + applications.createApplication(ApplicationId.defaultId()); + applications.createPutTransaction(ApplicationId.defaultId(), 1).commit(); + applications.reloadConfig(ApplicationSet.fromSingle(new Application(model, + new ServerCache(), + 1, + false, + vespaVersion, + MetricUpdater.createTestUpdater(), + ApplicationId.defaultId()))); + Set<ConfigKey<?>> configNames = applications.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); + assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); + + configNames = applications.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); + assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); } - private TenantApplications createZKAppRepo(MockReloadHandler reloadHandler) { - return TenantApplications.create(new TestComponentRegistry.Builder().curator(curator).build(), reloadHandler, tenantName); + @Test + public void testAppendIdsInNonRecursiveListing() { + assertEquals(applications.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); + assertEquals(applications.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); + assertEquals(applications.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); + assertEquals(applications.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); + } + + private TenantApplications createZKAppRepo() { + return TenantApplications.create(componentRegistry, tenantName); } private static ApplicationId createApplicationId(String name) { @@ -134,4 +222,10 @@ public class TenantApplicationsTest { .forPath(TenantRepository.getApplicationsPath(tenantName).append(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId)); } + + private ModelFactoryRegistry createRegistry() { + return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), + new TestModelFactory(new Version(3, 2, 1)))); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 2c119a119b6..a758698d3b5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -59,7 +59,7 @@ public class LocalSessionRepoTest { .build()) .build(); SessionFactory sessionFactory = new SessionFactory(globalComponentRegistry, - TenantApplications.create(globalComponentRegistry, new MockReloadHandler(), tenantName), + TenantApplications.create(globalComponentRegistry, tenantName), new HostRegistry<>(), tenantName); repo = new LocalSessionRepo(tenantName, globalComponentRegistry, sessionFactory); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index ce3b119d852..b072f20414f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; @@ -138,7 +137,7 @@ public class LocalSessionTest { File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); sessionDir.createNewFile(); TenantApplications applications = TenantApplications.create( - new TestComponentRegistry.Builder().curator(curator).build(), new MockReloadHandler(), tenant); + new TestComponentRegistry.Builder().curator(curator).build(), tenant); applications.createApplication(zkc.readApplicationId()); return new LocalSession(tenant, sessionId, preparer, FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, applications, new HostRegistry<>()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 876baa4cdda..51b0a36d8f4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -45,7 +46,7 @@ public class TenantRepositoryTest { private TenantRepository tenantRepository; private TestComponentRegistry globalComponentRegistry; - private TenantRequestHandlerTest.MockReloadListener listener; + private TenantApplicationsTest.MockReloadListener listener; private MockTenantListener tenantListener; private Curator curator; @@ -59,7 +60,7 @@ public class TenantRepositoryTest { public void setupSessions() { curator = new MockCurator(); globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - listener = (TenantRequestHandlerTest.MockReloadListener)globalComponentRegistry.getReloadListener(); + listener = (TenantApplicationsTest.MockReloadListener)globalComponentRegistry.getReloadListener(); tenantListener = (MockTenantListener)globalComponentRegistry.getTenantListener(); assertFalse(tenantListener.tenantsLoaded); tenantRepository = new TenantRepository(globalComponentRegistry); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java deleted file mode 100644 index fa9efec1255..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.component.Version; -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.DeployData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.protocol.DefContent; -import com.yahoo.vespa.config.protocol.VespaVersion; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.ServerCache; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; -import com.yahoo.vespa.config.server.model.TestModelFactory; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.VespaModelFactory; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.xml.sax.SAXException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class TenantRequestHandlerTest { - - private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); - private TenantRequestHandler server; - private MockReloadListener listener = new MockReloadListener(); - private File app1 = new File("src/test/apps/cs1"); - private File app2 = new File("src/test/apps/cs2"); - private TenantName tenant = TenantName.from("mytenant"); - private TestComponentRegistry componentRegistry; - private Curator curator; - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - - private ApplicationId defaultApp() { - return new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); - } - - @Before - public void setUp() throws IOException { - curator = new MockCurator(); - - feedApp(app1, 1, defaultApp(), false); - Metrics sh = Metrics.createTestMetrics(); - List<ReloadListener> listeners = new ArrayList<>(); - listeners.add(listener); - componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .modelFactoryRegistry(createRegistry()) - .build(); - server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), componentRegistry); - } - - private void feedApp(File appDir, long sessionId, ApplicationId appId, boolean internalRedeploy) throws IOException { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(appId); - File app = tempFolder.newFolder(); - IOUtils.copyDirectory(appDir, app); - ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); - DeployData deployData = new DeployData("user", - appDir.toString(), - appId, - 0L, - internalRedeploy, - 0L, - 0L); - ApplicationPackage appPackage = FilesApplicationPackage.fromFileWithDeployData(appDir, deployData); - deployer.deploy(appPackage, - Collections.singletonMap(vespaVersion, new MockFileRegistry()), - AllocatedHosts.withHosts(Collections.emptySet())); - } - - private ApplicationSet reloadConfig(long sessionId) { - return reloadConfig(sessionId, "default"); - } - - private ApplicationSet reloadConfig(long sessionId, String application) { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(new ApplicationId.Builder().tenant(tenant).applicationName(application).build()); - RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); - return session.ensureApplicationLoaded(); - } - - private ModelFactoryRegistry createRegistry() { - return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), - new TestModelFactory(new Version(3, 2, 1)))); - } - - private <T extends ConfigInstance> T resolve(Class<T> clazz, - TenantRequestHandler tenantRequestHandler, - ApplicationId appId, - Version vespaVersion, - String configId) { - ConfigResponse response = getConfigResponse(clazz, tenantRequestHandler, appId, vespaVersion, configId); - return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); - } - - private <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, - TenantRequestHandler tenantRequestHandler, - ApplicationId appId, - Version vespaVersion, - String configId) { - return tenantRequestHandler.resolveConfig(appId, new GetConfigRequest() { - @Override - public ConfigKey<T> getConfigKey() { - return new ConfigKey<>(clazz, configId); - } - - @Override - public DefContent getDefContent() { - return DefContent.fromClass(clazz); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return Optional.of(VespaVersion.fromString(vespaVersion.toFullString())); - } - - @Override - public boolean noCache() { - return false; - } - }, Optional.empty()); - } - - @Test - public void testReloadConfig() throws IOException { - ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); - - server.applications().createApplication(applicationId); - server.applications().createPutTransaction(applicationId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(1)); - // Using only payload list for this simple test - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertThat(config.intval(), is(1337)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1L)); - - server.reloadConfig(reloadConfig(1L)); - ConfigResponse configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertFalse(configResponse.isInternalRedeploy()); - config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertThat(config.intval(), is(1337)); - assertThat(listener.reloaded.get(), is(2)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1L)); - assertThat(listener.tenantHosts.size(), is(1)); - assertThat(server.resolveApplicationId("mytesthost"), is(applicationId)); - - listener.reloaded.set(0); - feedApp(app2, 2, defaultApp(), true); - server.applications().createPutTransaction(applicationId, 2).commit(); - server.reloadConfig(reloadConfig(2L)); - configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertTrue(configResponse.isInternalRedeploy()); - config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion,""); - assertThat(config.intval(), is(1330)); - assertThat(listener.reloaded.get(), is(1)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(2L)); - } - - @Test - public void testRemoveApplication() { - ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(0)); - - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(1)); - - assertThat(listener.removed.get(), is(0)); - - server.removeApplication(appId); - assertThat(listener.removed.get(), is(0)); - - server.applications().createDeleteTransaction(appId).commit(); - server.removeApplication(appId); - assertThat(listener.removed.get(), is(1)); - } - - @Test - public void testResolveForAppId() { - long id = 1L; - ApplicationId appId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp").instanceName("myinst").build(); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id))); - zkc.writeApplicationId(appId); - RemoteSession session = new RemoteSession(appId.tenant(), id, componentRegistry, zkc); - server.reloadConfig(session.ensureApplicationLoaded()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - } - - @Test - public void testResolveMultipleApps() throws IOException { - ApplicationId appId1 = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp1").instanceName("myinst1").build(); - ApplicationId appId2 = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp2").instanceName("myinst2").build(); - feedAndReloadApp(app1, 1, appId1); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId1, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - - feedAndReloadApp(app2, 2, appId2); - config = resolve(SimpletypesConfig.class, server, appId2, vespaVersion, ""); - assertThat(config.intval(), is(1330)); - } - - @Test - public void testResolveMultipleVersions() throws IOException { - ApplicationId appId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp1").instanceName("myinst1").build(); - feedAndReloadApp(app1, 1, appId); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - config = resolve(SimpletypesConfig.class, server, appId, new Version(3, 2, 1), ""); - assertThat(config.intval(), is(1337)); - } - - private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, sessionId).commit(); - feedApp(appDir, sessionId, appId, false); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(appId); - RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); - server.reloadConfig(session.ensureApplicationLoaded()); - } - - public static class MockReloadListener implements ReloadListener { - AtomicInteger reloaded = new AtomicInteger(0); - AtomicInteger removed = new AtomicInteger(0); - Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); - - @Override - public void configActivated(ApplicationSet application) { - reloaded.incrementAndGet(); - } - - @Override - public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { - tenantHosts.put(tenant.value(), newHosts); - } - - @Override - public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { - } - - @Override - public void applicationRemoved(ApplicationId applicationId) { - removed.incrementAndGet(); - } - } - - @Test - public void testHasApplication() { - assertdefaultAppNotFound(); - ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertTrue(server.hasApplication(appId, Optional.of(vespaVersion))); - } - - private void assertdefaultAppNotFound() { - assertFalse(server.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); - } - - @Test - public void testMultipleApplicationsReload() { - ApplicationId appId = ApplicationId.from(tenant.value(), "foo", "default"); - assertdefaultAppNotFound(); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1, "foo")); - assertdefaultAppNotFound(); - assertTrue(server.hasApplication(appId, - Optional.of(vespaVersion))); - assertThat(server.resolveApplicationId("doesnotexist"), is(ApplicationId.defaultId())); - assertThat(server.resolveApplicationId("mytesthost"), is(new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foo").build())); // Host set in application package. - } - - @Test - public void testListConfigs() throws IOException, SAXException { - assertdefaultAppNotFound(); - - VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); - server.applications().createApplication(ApplicationId.defaultId()); - server.applications().createPutTransaction(ApplicationId.defaultId(), 1).commit(); - server.reloadConfig(ApplicationSet.fromSingle(new Application(model, - new ServerCache(), - 1, - false, - vespaVersion, - MetricUpdater.createTestUpdater(), - ApplicationId.defaultId()))); - Set<ConfigKey<?>> configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); - assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); - - configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); - assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); - } - - @Test - public void testAppendIdsInNonRecursiveListing() { - assertEquals(server.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); - assertEquals(server.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); - assertEquals(server.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); - assertEquals(server.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); - } -} |