diff options
Diffstat (limited to 'configserver')
20 files changed, 365 insertions, 304 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index ab825f14b68..d0da4870f4f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -58,6 +58,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,6 +66,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * The API for managing applications. @@ -151,10 +153,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, boolean ignoreLockFailure, boolean ignoreSessionStaleFailure, Instant now) { - File tempDir = Files.createTempDir(); + return deploy(decompressApplication(in), prepareParams, ignoreLockFailure, ignoreSessionStaleFailure, now); + } + + public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { + return deploy(applicationPackage, prepareParams, false, false, Instant.now()); + } + + public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, + boolean ignoreLockFailure, boolean ignoreSessionStaleFailure, Instant now) { ApplicationId applicationId = prepareParams.getApplicationId(); - long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), decompressApplication(in, tempDir)); - cleanupApplicationDirectory(tempDir, logger); + long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); return prepareAndActivate(tenant, sessionId, prepareParams, ignoreLockFailure, ignoreSessionStaleFailure, now); } @@ -339,10 +348,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, InputStream in, String contentType) { - File tempDir = Files.createTempDir(); - long sessionId = createSession(applicationId, timeoutBudget, decompressApplication(in, contentType, tempDir)); - cleanupApplicationDirectory(tempDir, logger); - return sessionId; + return createSession(applicationId, timeoutBudget, decompressApplication(in, contentType)); } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { @@ -354,6 +360,32 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session.getSessionId(); } + + // ---------------- Tenant operations ---------------------------------------------------------------- + + public Set<TenantName> removeUnusedTenants() { + Set<TenantName> tenantsToBeDeleted = tenantRepository.getAllTenantNames().stream() + .filter(tenantName -> activeApplications(tenantName).isEmpty()) + .filter(tenantName -> !tenantName.equals(TenantName.defaultName())) // Not allowed to remove 'default' tenant + .collect(Collectors.toSet()); + tenantsToBeDeleted.forEach(tenantRepository::deleteTenant); + return tenantsToBeDeleted; + } + + public void deleteTenant(TenantName tenantName) { + List<ApplicationId> activeApplications = activeApplications(tenantName); + if (activeApplications.isEmpty()) + tenantRepository.deleteTenant(tenantName); + else + throw new IllegalArgumentException("Cannot delete tenant '" + tenantName + "', it has active applications: " + activeApplications); + } + + private List<ApplicationId> activeApplications(TenantName tenantName) { + return tenantRepository.getTenant(tenantName).getApplicationRepo().listApplications(); + } + + // ---------------- Misc operations ---------------------------------------------------------------- + public Tenant verifyTenantAndApplication(ApplicationId applicationId) { TenantName tenantName = applicationId.tenant(); if (!tenantRepository.checkThatTenantExists(tenantName)) { @@ -407,19 +439,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return currentActiveApplicationSet; } - private File decompressApplication(InputStream in, String contentType, File tempDir) { + private File decompressApplication(InputStream in, String contentType) { try (CompressedApplicationInputStream application = CompressedApplicationInputStream.createFromCompressedStream(in, contentType)) { - return decompressApplication(application, tempDir); + return decompressApplication(application); } catch (IOException e) { throw new IllegalArgumentException("Unable to decompress data in body", e); } } - private File decompressApplication(CompressedApplicationInputStream in, File tempDir) { + private File decompressApplication(CompressedApplicationInputStream in) { + File tempDir = Files.createTempDir(); try { return in.decompress(tempDir); } catch (IOException e) { + cleanupTempDirectory(tempDir, logger); throw new IllegalArgumentException("Unable to decompress stream", e); } } @@ -429,7 +463,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return applicationRepo.listApplications(); } - private static void cleanupApplicationDirectory(File tempDir, DeployLogger logger) { + private static void cleanupTempDirectory(File tempDir, DeployLogger logger) { logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index 27306289a9e..35c1e4fee80 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -39,8 +39,9 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac private final Curator curator; private final Path applicationsPath; - private final ExecutorService pathChildrenExecutor = - Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(ZKTenantApplications.class.getName())); + // One thread pool for all instances of this class + private static final ExecutorService pathChildrenExecutor = + Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(ZKTenantApplications.class.getName())); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final TenantName tenant; @@ -119,11 +120,10 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac @Override public void close() { directoryCache.close(); - pathChildrenExecutor.shutdown(); } @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { switch (event.getType()) { case CHILD_ADDED: applicationAdded(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java index 3857fea9d14..c8e9da1265b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -1,18 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import java.util.List; import com.google.inject.Inject; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.yolean.Exceptions; -import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.InternalServerException; @@ -27,11 +24,13 @@ public class TenantHandler extends HttpHandler { private static final String TENANT_NAME_REGEXP = "[\\w-]+"; private final TenantRepository tenantRepository; + private final ApplicationRepository applicationRepository; @Inject - public TenantHandler(HttpHandler.Context ctx, TenantRepository tenantRepository) { + public TenantHandler(Context ctx, TenantRepository tenantRepository, ApplicationRepository applicationRepository) { super(ctx); this.tenantRepository = tenantRepository; + this.applicationRepository = applicationRepository; } @Override @@ -62,22 +61,7 @@ public class TenantHandler extends HttpHandler { protected HttpResponse handleDELETE(HttpRequest request) { final TenantName tenantName = getTenantNameFromRequest(request); Utils.checkThatTenantExists(tenantRepository, tenantName); - // TODO: Move logic to ApplicationRepository - Tenant tenant = tenantRepository.getTenant(tenantName); - TenantApplications applicationRepo = tenant.getApplicationRepo(); - final List<ApplicationId> activeApplications = applicationRepo.listApplications(); - if (activeApplications.isEmpty()) { - try { - tenantRepository.deleteTenant(tenantName); - } catch (IllegalArgumentException e) { - throw e; - } catch (Exception e) { - throw new InternalServerException(Exceptions.toMessageString(e)); - } - } else { - throw new BadRequestException("Cannot delete tenant '" + tenantName + "', as it has active applications: " + - activeApplications); - } + applicationRepository.deleteTenant(tenantName); return new TenantDeleteResponse(tenantName); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java new file mode 100644 index 00000000000..c8b3bc824a8 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; + +public class ConfigServerMaintenance extends AbstractComponent { + + private final TenantsMaintainer tenantsMaintainer; + + @SuppressWarnings("unused") // instantiated by Dependency Injection + public ConfigServerMaintenance(ConfigserverConfig configserverConfig, + ApplicationRepository applicationRepository, + Curator curator) { + tenantsMaintainer = new TenantsMaintainer(applicationRepository, + curator, + Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes())); + } + + @Override + public void deconstruct() { + tenantsMaintainer.deconstruct(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java new file mode 100644 index 00000000000..ce0811184a3 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java @@ -0,0 +1,73 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; + +import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +public abstract class Maintainer extends AbstractComponent implements Runnable { + + protected static final Logger log = Logger.getLogger(Maintainer.class.getName()); + private static final Path root = Path.fromString("/configserver/v1/"); + private static final com.yahoo.path.Path lockRoot = root.append("locks"); + + private final Duration maintenanceInterval; + private final ScheduledExecutorService service; + protected final ApplicationRepository applicationRepository; + protected final Curator curator; + + Maintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + this.applicationRepository = applicationRepository; + this.curator = curator; + this.maintenanceInterval = interval; + service = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory(name())); + service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); + } + + @Override + @SuppressWarnings("try") + public void run() { + try { + Path path = lockRoot.append(name()); + try (Lock lock = new Lock(path.toString(), curator)) { + maintain(); + } + } catch (UncheckedTimeoutException e) { + // another config server instance is running this job at the moment; ok + } catch (Throwable t) { + log.log(Level.WARNING, this + " failed. Will retry in " + maintenanceInterval.toMinutes() + " minutes", t); + } + } + + @Override + public void deconstruct() { + this.service.shutdown(); + } + + /** + * Called once each time this maintenance job should run + */ + protected abstract void maintain(); + + public String name() { return this.getClass().getSimpleName(); } + + /** + * Returns the name of this + */ + @Override + public final String toString() { + return name(); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java new file mode 100644 index 00000000000..e06bf530486 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -0,0 +1,19 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; + +public class TenantsMaintainer extends Maintainer { + + public TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, interval); + } + + @Override + protected void maintain() { + applicationRepository.removeUnusedTenants(); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index 6a3e9c77809..550b08f3d5c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.application.ZKTenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import java.io.File; @@ -10,6 +12,8 @@ import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -26,7 +30,9 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); private static final Duration delay = Duration.ofMinutes(5); - private final ScheduledExecutorService purgeOldSessionsExecutor = new ScheduledThreadPoolExecutor(1); + // One executor for all instances of this class + private static final ScheduledExecutorService purgeOldSessionsExecutor = + new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getDaemonThreadFactory("purge-old-sessions")); private final long sessionLifetime; // in seconds private final Clock clock; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index b71e85021c3..12fa828f692 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -3,12 +3,14 @@ package com.yahoo.vespa.config.server.session; import java.util.*; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; +import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -38,6 +40,10 @@ import org.apache.curator.framework.recipes.cache.*; public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements NodeCacheListener, PathChildrenCacheListener { private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); + // One thread pool for all instances of this class + private static final ExecutorService pathChildrenExecutor = + Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(RemoteSessionRepo.class.getName())); + private final Curator curator; private final Path sessionsPath; private final RemoteSessionFactory remoteSessionFactory; @@ -53,15 +59,13 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod * @param reloadHandler a {@link com.yahoo.vespa.config.server.ReloadHandler} * @param tenant a {@link TenantName} instance. * @param applicationRepo a {@link TenantApplications} instance. - * @param executorService an {@link ExecutorService} to run callbacks from ZooKeeper. */ public RemoteSessionRepo(Curator curator, RemoteSessionFactory remoteSessionFactory, ReloadHandler reloadHandler, TenantName tenant, TenantApplications applicationRepo, - MetricUpdater metricUpdater, - ExecutorService executorService) { + MetricUpdater metricUpdater) { this.curator = curator; this.sessionsPath = TenantRepository.getSessionsPath(tenant); this.applicationRepo = applicationRepo; @@ -69,7 +73,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod this.reloadHandler = reloadHandler; this.metrics = metricUpdater; initializeSessions(); - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, executorService); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, pathChildrenExecutor); this.directoryCache.addListener(this); this.directoryCache.start(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index cd437704f9a..69721ed01d4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.path.Path; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -20,8 +19,6 @@ import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.time.Clock; import java.util.Collections; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Builder for helping out with tenant creation. Each of a tenants dependencies may be overridden for testing. @@ -159,17 +156,10 @@ public class TenantBuilder { reloadHandler, tenant, applicationRepo, - componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant)), - // TODO: Check if we can avoid using one executor service per tenant. Either one for all - // or have a few executors and hash on tenant name to find out which one to use here - createSingleThreadedExecutorService(RemoteSessionRepo.class.getName())); + componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenant))); } } - private ExecutorService createSingleThreadedExecutorService(String executorName) { - return Executors.newSingleThreadExecutor(ThreadFactoryFactory.getThreadFactory(executorName + "-" + tenant.value())); - } - private void createServerDbDirs() { if (tenantFileSystemDirs == null) { tenantFileSystemDirs = new TenantFileSystemDirs(new File( diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index a9e67738d96..b984ce60702 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -40,6 +40,7 @@ <component id="com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.filedistribution.FileServer" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.maintenance.ConfigServerMaintenance" bundle="configserver" /> <component id="com.yahoo.vespa.serviceview.ConfigServerLocation" bundle="configserver" /> diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 8c0ea810a9a..03ce136e13c 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -56,9 +56,12 @@ findroot () { findhost () { if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname) || exit 1 + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 fi - vespa-validate-hostname "${VESPA_HOSTNAME}" || exit 1 export VESPA_HOSTNAME } @@ -95,10 +98,19 @@ fixddir ${VESPA_HOME}/var/zookeeper fixfile ${VESPA_HOME}/var/zookeeper/myid fixddir ${VESPA_HOME}/var/zookeeper/version-2 -${VESPA_HOME}/libexec/vespa/vespa-config.pl -isthisaconfigserver 1>/dev/null -if [ "$?" != "0" ] ; then - echo "Not able to start config server, host `hostname` is not part of 'VESPA_CONFIGSERVERS'" - exit 1; +not_a_configserver () { + for hn in $(vespa-print-default configservers); do + if [ "$hn" = localhost ] || [ "$hn" = "${VESPA_HOSTNAME}" ]; then + return 1 + fi + done + return 0 +} + +if not_a_configserver ; then + echo "Will not start config server, host ${VESPA_HOSTNAME}" \ + "is not part of VESPA_CONFIGSERVERS: " $(vespa-print-default configservers) + exit 1 fi fixlimits @@ -139,7 +151,7 @@ baseuserargs="$vespa_base__jvmargs_configserver" serveruserargs="$cloudconfig_server__jvmargs" jvmargs="$baseuserargs $serveruserargs" -export LD_PRELOAD=${VESPA_HOME}/lib64/vespa/malloc/libvespamallocd.so +export LD_PRELOAD=${VESPA_HOME}/lib64/vespa/malloc/libvespamalloc.so printenv > $cfpfile fixddir $bundlecachedir diff --git a/configserver/src/main/sh/start-logd b/configserver/src/main/sh/start-logd index ce0b60a8d8d..2b6ed01f09c 100644 --- a/configserver/src/main/sh/start-logd +++ b/configserver/src/main/sh/start-logd @@ -56,9 +56,12 @@ findroot () { findhost () { if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname) || exit 1 + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 fi - vespa-validate-hostname "${VESPA_HOSTNAME}" || exit 1 export VESPA_HOSTNAME } diff --git a/configserver/src/main/sh/stop-configserver b/configserver/src/main/sh/stop-configserver index 957fdb249c3..099429eb556 100755 --- a/configserver/src/main/sh/stop-configserver +++ b/configserver/src/main/sh/stop-configserver @@ -56,9 +56,12 @@ findroot () { findhost () { if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname) || exit 1 + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 fi - vespa-validate-hostname "${VESPA_HOSTNAME}" || exit 1 export VESPA_HOSTNAME } diff --git a/configserver/src/main/sh/vespa-configserver-remove-state b/configserver/src/main/sh/vespa-configserver-remove-state index e3d348ecefa..faac37d48d4 100755 --- a/configserver/src/main/sh/vespa-configserver-remove-state +++ b/configserver/src/main/sh/vespa-configserver-remove-state @@ -56,9 +56,12 @@ findroot () { findhost () { if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname) || exit 1 + VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 + fi + validate="${VESPA_HOME}/bin/vespa-validate-hostname" + if [ -f "$validate" ]; then + "$validate" "${VESPA_HOSTNAME}" || exit 1 fi - vespa-validate-hostname "${VESPA_HOSTNAME}" || exit 1 export VESPA_HOSTNAME } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 90f0b5ee4e5..17cbe41fde5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -27,6 +27,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -76,18 +77,26 @@ public class ApplicationRepositoryTest { @Test public void createAndPrepareAndActivate() throws IOException { - PrepareResult result = createAndPrepareAndActivateApp(); + PrepareResult result = deployApp(); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); } + @Test + public void deleteUnusedTenants() throws IOException { + deployApp(); + assertTrue(applicationRepository.removeUnusedTenants().isEmpty()); + applicationRepository.remove(applicationId()); + assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next()); + } + private PrepareResult prepareAndActivateApp(File application) throws IOException { FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir()); return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now()); } - private PrepareResult createAndPrepareAndActivateApp() throws IOException { + private PrepareResult deployApp() throws IOException { File file = CompressedApplicationInputStreamTest.createTarFile(); return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream( new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 0a193f7eedd..ce84cf4c280 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -3,21 +3,14 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; -import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.SuperModelGenerationCounter; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; import com.yahoo.vespa.config.server.application.HttpProxy; @@ -25,22 +18,13 @@ import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.StaticResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SimpleHttpFetcher; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.MockSessionZKClient; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.SessionContext; -import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; +import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.model.VespaModelFactory; -import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import javax.ws.rs.client.Client; @@ -48,7 +32,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.time.Clock; -import java.util.Collections; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; @@ -69,14 +52,14 @@ public class ApplicationHandlerTest { private static File testApp = new File("src/test/apps/app"); - private ApplicationHandler mockHandler; private ListApplicationsHandler listApplicationsHandler; private final static TenantName mytenantName = TenantName.from("mytenant"); private final static TenantName foobar = TenantName.from("foobar"); + private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); private TenantRepository tenantRepository; + private ApplicationRepository applicationRepository; private SessionHandlerTest.MockProvisioner provisioner; private MockStateApiFactory stateApiFactory = new MockStateApiFactory(); - private final HttpProxy mockHttpProxy = mock(HttpProxy.class); @Before public void setup() { @@ -85,136 +68,103 @@ public class ApplicationHandlerTest { tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); provisioner = new SessionHandlerTest.MockProvisioner(); - mockHandler = createMockApplicationHandler(provisioner, - new ApplicationConvergenceChecker(stateApiFactory), - mockHttpProxy); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, Clock.systemUTC()); listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), tenantRepository, Zone.defaultZone()); } - private ApplicationHandler createMockApplicationHandler( - Provisioner provisioner, - ApplicationConvergenceChecker convergeChecker, - HttpProxy httpProxy) { - return new ApplicationHandler( - ApplicationHandler.testOnlyContext(), - Zone.defaultZone(), - new ApplicationRepository(tenantRepository, - HostProvisionerProvider.withProvisioner(provisioner), - convergeChecker, - httpProxy, - new ConfigserverConfig(new ConfigserverConfig.Builder()))); - } - - private ApplicationHandler createApplicationHandler(TenantRepository tenantRepository) { - return new ApplicationHandler( - ApplicationHandler.testOnlyContext(), - Zone.defaultZone(), - new ApplicationRepository(tenantRepository, - HostProvisionerProvider.withProvisioner(provisioner), - new ApplicationConvergenceChecker(stateApiFactory), - new HttpProxy(new SimpleHttpFetcher()), - new ConfigserverConfig(new ConfigserverConfig.Builder()))); - } - @Test public void testDelete() throws Exception { - Clock clock = Clock.systemUTC(); - ApplicationId defaultId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - assertApplicationExists(mytenantName, null, Zone.defaultZone()); - - long sessionId = 1; { // This block is a real test of the interplay of (most of) the components of the config server // TODO: Extract it to ApplicationRepositoryTest, rewrite to bypass the HTTP layer and extend // as login is moved from the HTTP layer into ApplicationRepository - TenantRepository tenantRepository = addApplication(defaultId, sessionId); - ApplicationHandler handler = createApplicationHandler(tenantRepository); - Tenant mytenant = tenantRepository.getTenant(defaultId.tenant()); + + PrepareResult result = applicationRepository.deploy(testApp, prepareParams(applicationId)); + long sessionId = result.sessionId(); + Tenant mytenant = tenantRepository.getTenant(applicationId.tenant()); LocalSession applicationData = mytenant.getLocalSessionRepo().getSession(sessionId); assertNotNull(applicationData); assertNotNull(applicationData.getApplicationId()); assertFalse(provisioner.removed); - deleteAndAssertOKResponse(handler, mytenant, defaultId); + deleteAndAssertOKResponse(mytenant, applicationId); assertTrue(provisioner.removed); assertThat(provisioner.lastApplicationId.tenant(), is(mytenantName)); - assertThat(provisioner.lastApplicationId, is(defaultId)); + assertThat(provisioner.lastApplicationId, is(applicationId)); assertNull(mytenant.getLocalSessionRepo().getSession(sessionId)); assertNull(mytenant.getRemoteSessionRepo().getSession(sessionId)); } - sessionId++; { - addMockApplication(tenantRepository.getTenant(mytenantName), defaultId, sessionId, clock); - deleteAndAssertOKResponseMocked(defaultId, true); + applicationRepository.deploy(testApp, prepareParams(applicationId)); + deleteAndAssertOKResponseMocked(applicationId, true); + applicationRepository.deploy(testApp, prepareParams(applicationId)); ApplicationId fooId = new ApplicationId.Builder() - .tenant(mytenantName) - .applicationName("foo").instanceName("quux").build(); - - sessionId++; - - addMockApplication(tenantRepository.getTenant(mytenantName), fooId, sessionId, clock); - addMockApplication(tenantRepository.getTenant(foobar), fooId, sessionId, clock); - assertApplicationExists(mytenantName, fooId, Zone.defaultZone()); - assertApplicationExists(foobar, fooId, Zone.defaultZone()); + .tenant(foobar) + .applicationName("foo") + .instanceName("quux") + .build(); + PrepareParams prepareParams2 = new PrepareParams.Builder().applicationId(fooId).build(); + applicationRepository.deploy(testApp, prepareParams2); + + assertApplicationExists(fooId, Zone.defaultZone()); deleteAndAssertOKResponseMocked(fooId, true); - assertThat(provisioner.lastApplicationId.tenant(), is(mytenantName)); assertThat(provisioner.lastApplicationId, is(fooId)); - assertApplicationExists(mytenantName, null, Zone.defaultZone()); - assertApplicationExists(foobar, fooId, Zone.defaultZone()); + assertApplicationExists(applicationId, Zone.defaultZone()); + + deleteAndAssertOKResponseMocked(applicationId, true); } - sessionId++; { ApplicationId baliId = new ApplicationId.Builder() .tenant(mytenantName) - .applicationName("bali").instanceName("quux").build(); - addMockApplication(tenantRepository.getTenant(mytenantName), baliId, sessionId, clock); + .applicationName("bali") + .instanceName("quux") + .build(); + PrepareParams prepareParamsBali = new PrepareParams.Builder().applicationId(baliId).build(); + applicationRepository.deploy(testApp, prepareParamsBali); deleteAndAssertOKResponseMocked(baliId, true); - assertApplicationExists(mytenantName, null, Zone.defaultZone()); } } @Test public void testGet() throws Exception { - long sessionId = 1; - ApplicationId defaultId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - addMockApplication(tenantRepository.getTenant(mytenantName), defaultId, sessionId, Clock.systemUTC()); - assertApplicationGeneration(defaultId, Zone.defaultZone(), 1, true); - assertApplicationGeneration(defaultId, Zone.defaultZone(), 1, false); + long sessionId = applicationRepository.deploy(testApp, prepareParams(applicationId)).sessionId(); + assertApplicationGeneration(applicationId, Zone.defaultZone(), sessionId, true); + assertApplicationGeneration(applicationId, Zone.defaultZone(), sessionId, false); } @Test public void testRestart() throws Exception { - long sessionId = 1; - ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - addMockApplication(tenantRepository.getTenant(mytenantName), application, sessionId, Clock.systemUTC()); + applicationRepository.deploy(testApp, prepareParams(applicationId)); assertFalse(provisioner.restarted); - restart(application, Zone.defaultZone()); + restart(applicationId, Zone.defaultZone()); assertTrue(provisioner.restarted); - assertEquals(application, provisioner.lastApplicationId); + assertEquals(applicationId, provisioner.lastApplicationId); } @Test public void testConverge() throws Exception { - long sessionId = 1; - ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - addMockApplication(tenantRepository.getTenant(mytenantName), application, sessionId, Clock.systemUTC()); - converge(application, Zone.defaultZone()); + applicationRepository.deploy(testApp, prepareParams(applicationId)); + converge(applicationId, Zone.defaultZone()); } @Test public void testClusterControllerStatus() throws Exception { - long sessionId = 1; - ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - addMockApplication(tenantRepository.getTenant(mytenantName), application, sessionId, Clock.systemUTC()); + applicationRepository.deploy(testApp, prepareParams(applicationId)); String host = "foo.yahoo.com"; - String url = toUrlPath(application, Zone.defaultZone(), true) + "/clustercontroller/" + host + "/status/v1/clusterName1"; - + String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/clustercontroller/" + host + "/status/v1/clusterName1"; + HttpProxy mockHttpProxy = mock(HttpProxy.class); + ApplicationRepository applicationRepository = new ApplicationRepository(tenantRepository, + HostProvisionerProvider.withProvisioner(provisioner), + new ApplicationConvergenceChecker(stateApiFactory), + mockHttpProxy, + new ConfigserverConfig(new ConfigserverConfig.Builder())); + ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq("container-clustercontroller"), eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); @@ -228,42 +178,11 @@ public class ApplicationHandlerTest { } @Test - @Ignore - public void testFailingProvisioner() throws Exception { - provisioner = new SessionHandlerTest.FailingMockProvisioner(); - mockHandler = createMockApplicationHandler( - provisioner, - new ApplicationConvergenceChecker(stateApiFactory), - new HttpProxy(new SimpleHttpFetcher())); - final ApplicationId applicationId = ApplicationId.defaultId(); - addMockApplication(tenantRepository.getTenant(mytenantName), applicationId, 1, Clock.systemUTC()); - assertApplicationExists(mytenantName, applicationId, Zone.defaultZone()); - provisioner.activated = true; - - String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/" + applicationId.application(); - deleteAndAssertResponse(mockHandler, url, 500, null, "{\"message\":\"Cannot remove application\"}", com.yahoo.jdisc.http.HttpRequest.Method.DELETE); - assertApplicationExists(mytenantName, applicationId, Zone.defaultZone()); - Assert.assertTrue(provisioner.activated); - } - - static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId, Clock clock) { - tenant.getApplicationRepo().createPutApplicationTransaction(applicationId, sessionId).commit(); - ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) - .build(); - tenant.getRemoteSessionRepo().addSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app), clock)); - } - - @Test public void testFileDistributionStatus() throws Exception { - long sessionId = 1; - ApplicationId application = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - addMockApplication(tenantRepository.getTenant(mytenantName), application, sessionId, Clock.systemUTC()); + applicationRepository.deploy(testApp, prepareParams(applicationId)); Zone zone = Zone.defaultZone(); - HttpResponse response = fileDistributionStatus(application, zone); + HttpResponse response = fileDistributionStatus(applicationId, zone); assertEquals(200, response.getStatus()); SessionHandlerTest.getRenderedString(response); assertEquals("{\"hosts\":[{\"hostname\":\"mytesthost\",\"status\":\"UNKNOWN\",\"message\":\"error: Connection error(104)\",\"fileReferences\":[]}],\"status\":\"UNKNOWN\"}", @@ -277,71 +196,31 @@ public class ApplicationHandlerTest { SessionHandlerTest.getRenderedString(responseForUnknown)); } - private static TenantRepository addApplication(ApplicationId applicationId, long sessionId) throws Exception { - // This method is a good illustration of the spaghetti wiring resulting from no design - // TODO: When this setup looks sane we have refactored sufficiently that there is a design - - TenantName tenantName = applicationId.tenant(); - - Path tenantPath = TenantRepository.getTenantPath(tenantName); - Path sessionPath = tenantPath.append(Tenant.SESSIONS).append(String.valueOf(sessionId)); - - MockCurator curator = new MockCurator(); - GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .modelFactoryRegistry(new ModelFactoryRegistry( - Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) - .build(); - - TenantRepository tenantRepository = new TenantRepository(componentRegistry); // Creates the application path element in zk - tenantRepository.addTenant(tenantName); - Tenant tenant = tenantRepository.getTenant(tenantName); - - tenant.getApplicationRepo().createPutApplicationTransaction(applicationId, sessionId).commit(); - ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - - SessionZooKeeperClient sessionClient = new SessionZooKeeperClient(curator, sessionPath); - SessionContext context = new SessionContext(app, - sessionClient, - new File("/serverDb"), - tenant.getApplicationRepo(), - null, - new SuperModelGenerationCounter(curator)); - tenant.getLocalSessionRepo().addSession(new LocalSession(tenantName, sessionId, null, context)); - sessionClient.writeApplicationId(applicationId); // TODO: Instead, use ApplicationRepository to deploy the application - - tenant.getRemoteSessionRepo().addSession( - new RemoteSession(tenantName, sessionId, - componentRegistry, - sessionClient, - Clock.systemUTC())); - return tenantRepository; - } - private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default"; - deleteAndAssertResponse(mockHandler, url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", + deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", method); } private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().getSessionIdForApplication(applicationId); - deleteAndAssertResponse(mockHandler, applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); + deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); assertNull(tenantRepository.getTenant(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); } - private void deleteAndAssertOKResponse(ApplicationHandler handler, Tenant tenant, ApplicationId applicationId) throws IOException { + private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { long sessionId = tenant.getApplicationRepo().getSessionIdForApplication(applicationId); - deleteAndAssertResponse(handler, applicationId, Zone.defaultZone(), Response.Status.OK, null, true); + deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, true); assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); } - private void deleteAndAssertResponse(ApplicationHandler handler, ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException { + private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException { String expectedResponse = "{\"message\":\"Application '" + applicationId + "' deleted\"}"; - deleteAndAssertResponse(handler, toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE); + deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE); } - private void deleteAndAssertResponse(ApplicationHandler handler, String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { + private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException { + ApplicationHandler handler = createApplicationHandler(); HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method)); if (expectedStatus == 200) { HandlerTest.assertHttpStatusCodeAndMessage(response, 200, expectedResponse); @@ -362,11 +241,12 @@ public class ApplicationHandlerTest { } private void assertApplicationGeneration(String url, long expectedGeneration) throws IOException { - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"generation\":" + expectedGeneration + "}"); } - private void assertApplicationExists(TenantName tenantName, ApplicationId applicationId, Zone zone) throws IOException { + private void assertApplicationExists(ApplicationId applicationId, Zone zone) throws IOException { + String tenantName = applicationId == null ? null : applicationId.tenant().value(); String expected = applicationId == null ? "[]" : "[\"http://myhost:14000/application/v2/tenant/" + tenantName + "/application/" + applicationId.application().value() + "/environment/" + zone.environment().value() + "/region/" + zone.region().value() + @@ -379,23 +259,23 @@ public class ApplicationHandlerTest { private void restart(ApplicationId application, Zone zone) throws IOException { String restartUrl = toUrlPath(application, zone, true) + "/restart"; - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.POST)); + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.POST)); HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); } private void converge(ApplicationId application, Zone zone) throws IOException { String convergeUrl = toUrlPath(application, zone, true) + "/serviceconverge"; - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(convergeUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(convergeUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); HandlerTest.assertHttpStatusCodeAndMessage(response, 200, ""); } private HttpResponse fileDistributionStatus(ApplicationId application, Zone zone) { String restartUrl = toUrlPath(application, zone, true) + "/filedistributionstatus"; - return mockHandler.handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + return createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.GET)); } private static class MockStateApiFactory implements ApplicationConvergenceChecker.StateApiFactory { - public boolean createdApi = false; + boolean createdApi = false; @Override public ApplicationConvergenceChecker.StateApi createStateApi(Client client, URI serviceUri) { createdApi = true; @@ -409,4 +289,16 @@ public class ApplicationHandlerTest { } } + private ApplicationHandler createApplicationHandler() { + return createApplicationHandler(applicationRepository); + } + + private ApplicationHandler createApplicationHandler(ApplicationRepository applicationRepository) { + return new ApplicationHandler(ApplicationHandler.testOnlyContext(), Zone.defaultZone(), applicationRepository); + } + + private PrepareParams prepareParams(ApplicationId applicationId) { + return new PrepareParams.Builder().applicationId(applicationId).build(); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 0ac08bfcfb0..04b187f6b1d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -1,6 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.*; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -10,11 +13,18 @@ import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.session.MockSessionZKClient; +import com.yahoo.vespa.config.server.session.RemoteSession; +import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.time.Clock; import java.util.Collections; @@ -28,6 +38,7 @@ import static org.junit.Assert.assertThat; */ public class HostHandlerTest { private static final String urlPrefix = "http://myhost:14000/application/v2/host/"; + private static File testApp = new File("src/test/apps/app"); private HostHandler handler; private final static TenantName mytenant = TenantName.from("mytenant"); @@ -36,6 +47,16 @@ public class HostHandlerTest { private HostRegistries hostRegistries; private HostHandler hostHandler; + static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId, Clock clock) { + tenant.getApplicationRepo().createPutApplicationTransaction(applicationId, sessionId).commit(); + ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); + tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) + .build(); + tenant.getRemoteSessionRepo().addSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app), clock)); + } + @Before public void setup() { TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); @@ -63,14 +84,14 @@ public class HostHandlerTest { assertThat(hostRegistries, is(hostHandler.hostRegistries)); long sessionId = 1; ApplicationId id = ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()); - ApplicationHandlerTest.addMockApplication(tenantRepository.getTenant(mytenant), id, sessionId, Clock.systemUTC()); + addMockApplication(tenantRepository.getTenant(mytenant), id, sessionId, Clock.systemUTC()); assertApplicationForHost(hostname, mytenant, id, Zone.defaultZone()); } @Test public void require_that_handler_gives_error_for_unknown_hostname() throws Exception { long sessionId = 1; - ApplicationHandlerTest.addMockApplication(tenantRepository.getTenant(mytenant), ApplicationId.defaultId(), sessionId, Clock.systemUTC()); + addMockApplication(tenantRepository.getTenant(mytenant), ApplicationId.defaultId(), sessionId, Clock.systemUTC()); final String hostname = "unknown"; assertErrorForHost(hostname, Response.Status.NOT_FOUND, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index 6e56d3c30c3..35b22d19d6a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.http.v2; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Clock; @@ -11,7 +12,14 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.http.SessionResponse; import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -20,14 +28,23 @@ import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.NotFoundException; -public class TenantHandlerTest extends TenantTest { +public class TenantHandlerTest { + private TenantRepository tenantRepository; private TenantHandler handler; private final TenantName a = TenantName.from("a"); @Before public void setup() { - handler = new TenantHandler(TenantHandler.testOnlyContext(), tenantRepository); + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder().curator(new MockCurator()).build()); + ApplicationRepository applicationRepository = + new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + handler = new TenantHandler(TenantHandler.testOnlyContext(), tenantRepository, applicationRepository); + } + + @After + public void closeTenantRepo() { + tenantRepository.close(); } @Test @@ -91,13 +108,13 @@ public class TenantHandlerTest extends TenantTest { int sessionId = 1; ApplicationId app = ApplicationId.from(a, ApplicationName.from("foo"), InstanceName.defaultName()); - ApplicationHandlerTest.addMockApplication(tenant, app, sessionId, Clock.systemUTC()); + HostHandlerTest.addMockApplication(tenant, app, sessionId, Clock.systemUTC()); try { handler.handleDELETE(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/" + a, Method.DELETE)); fail(); - } catch (BadRequestException e) { - assertThat(e.getMessage(), is("Cannot delete tenant 'a', as it has active applications: [a.foo]")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Cannot delete tenant 'a', it has active applications: [a.foo]")); } } @@ -115,4 +132,10 @@ public class TenantHandlerTest extends TenantTest { return (TenantCreateResponse) handler.handlePUT(testRequest); } + private void assertResponseEquals(SessionResponse response, String payload) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + response.render(baos); + assertEquals(baos.toString("UTF-8"), payload); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java deleted file mode 100644 index 7814266f815..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http.v2; - -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.concurrent.Executor; - -import com.yahoo.vespa.config.server.*; -import com.yahoo.vespa.config.server.http.SessionResponse; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.junit.After; -import org.junit.Before; - -/** - * Supertype for tests in the multi tenant application API - * - * @author Vegard Havdal - * - */ -public class TenantTest extends TestWithCurator { - - protected TenantRepository tenantRepository; - - @Before - public void setupTenants() { - tenantRepository = createTenants(); - } - - @After - public void closeTenants() { - tenantRepository.close(); - } - - private TenantRepository createTenants() { - return new TenantRepository(new TestComponentRegistry.Builder().curator(curator).build()); - } - - void assertResponseEquals(SessionResponse response, String payload) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals(baos.toString("UTF-8"), payload); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 3598af57593..42c2d8db968 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -129,7 +129,8 @@ public class TenantRepositoryTest extends TestWithCurator { @Test public void testTenantsChanged() { - tenantRepository = new TenantRepository(globalComponentRegistry); + tenantRepository.close(); // Close the repo created in setup() + TenantRepository tenantRepository = new TenantRepository(globalComponentRegistry); tenantRepository.addTenant(tenant2); tenantRepository.createTenants(); Set<TenantName> allTenants = tenantRepository.getAllTenantNames(); |