diff options
| author | Jon Bratseth <bratseth@gmail.com> | 2020-09-28 17:38:19 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@gmail.com> | 2020-09-28 17:38:19 +0200 |
| commit | b12c17ea5e8a738570bc897e0fcd1e7f19688688 (patch) | |
| tree | aa48cbfe679e9ca1c5b83812e8e6cf45257784ab /configserver | |
| parent | af1d10a394b55cc0d31575bf979cecbbc177c568 (diff) | |
| parent | 22e2f5ad80aedfe8ed3a9af7c7c11244ef184ed3 (diff) | |
Merge branch 'master' into bratseth/autoscaling-reconfiguration-events
Diffstat (limited to 'configserver')
60 files changed, 813 insertions, 735 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3564a6e6da7..4670184c303 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -24,7 +24,6 @@ import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.path.Path; -import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.application.Application; @@ -65,7 +64,10 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -128,6 +130,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final LogRetriever logRetriever; private final TesterClient testerClient; private final Metric metric; + private final BooleanFlag deployWithInternalRestart; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -177,6 +180,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.clock = Objects.requireNonNull(clock); this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); + this.deployWithInternalRestart = Flags.DEPLOY_WITH_INTERNAL_RESTART.bindTo(flagSource); } public static class Builder { @@ -280,28 +284,29 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye bootstrapping.set(false); } - public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, Instant now) { + public PrepareResult prepare(Tenant tenant, long sessionId, PrepareParams prepareParams) { + DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams); + Deployment deployment = prepare(tenant, sessionId, prepareParams, logger); + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); + } + + private Deployment prepare(Tenant tenant, long sessionId, PrepareParams prepareParams, DeployHandlerLogger logger) { validateThatLocalSessionIsNotActive(tenant, sessionId); LocalSession session = getLocalSession(tenant, sessionId); ApplicationId applicationId = prepareParams.getApplicationId(); - Optional<ApplicationSet> currentActiveApplicationSet = getCurrentActiveApplicationSet(tenant, applicationId); - Slime deployLog = createDeployLog(); - DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); - try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { - SessionRepository sessionRepository = tenant.getSessionRepository(); - ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, - currentActiveApplicationSet, tenant.getPath(), now); - logConfigChangeActions(actions, logger); - log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); - return new PrepareResult(sessionId, actions, deployLog); - } + Deployment deployment = Deployment.unprepared(session, this, hostProvisioner, tenant, prepareParams, logger, clock); + deployment.prepare(); + + logConfigChangeActions(deployment.configChangeActions(), logger); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + return deployment; } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, Instant now) { + public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); PrepareResult prepareResult; try { - prepareResult = deploy(decompressApplication(in, tempDir), prepareParams, now); + prepareResult = deploy(decompressApplication(in, tempDir), prepareParams); } finally { cleanupTempDirectory(tempDir); } @@ -309,16 +314,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams) { - return deploy(applicationPackage, prepareParams, Instant.now()); - } - - public PrepareResult deploy(File applicationPackage, PrepareParams prepareParams, Instant now) { ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); Tenant tenant = getTenant(applicationId); - PrepareResult result = prepare(tenant, sessionId, prepareParams, now); - activate(tenant, sessionId, prepareParams.getTimeoutBudget(), prepareParams.force()); - return result; + DeployHandlerLogger logger = DeployHandlerLogger.forPrepareParams(prepareParams); + Deployment deployment = prepare(tenant, sessionId, prepareParams, logger); + deployment.activate(); + + return new PrepareResult(sessionId, deployment.configChangeActions(), logger); } /** @@ -379,9 +382,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye SessionRepository sessionRepository = tenant.getSessionRepository(); LocalSession newSession = sessionRepository.createSessionFromExisting(activeSession, logger, true, timeoutBudget); sessionRepository.addLocalSession(newSession); + boolean internalRestart = deployWithInternalRestart.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, - false /* don't validate as this is already deployed */, bootstrap)); + return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock, + false /* don't validate as this is already deployed */, bootstrap, internalRestart)); } @Override @@ -398,15 +402,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye TimeoutBudget timeoutBudget, boolean force) { LocalSession localSession = getLocalSession(tenant, sessionId); - Deployment deployment = deployment(localSession, tenant, timeoutBudget.timeLeft(), force); + Deployment deployment = Deployment.prepared(localSession, this, hostProvisioner, tenant, logger, timeoutBudget.timeout(), clock, false, force); deployment.activate(); return localSession.getApplicationId(); } - private Deployment deployment(LocalSession session, Tenant tenant, Duration timeout, boolean force) { - return Deployment.prepared(session, this, hostProvisioner, tenant, timeout, clock, false, force); - } - public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean force) { Tenant tenant = tenantRepository.getTenant(prepared.getTenantName()); Transaction transaction = tenant.getSessionRepository().createActivateTransaction(prepared); @@ -520,8 +520,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(new ApplicationRolesStore(curator, tenant.getPath()).delete(applicationId)); // Delete endpoint certificates transaction.add(new EndpointCertificateMetadataStore(curator, tenant.getPath()).delete(applicationId)); - // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server - // this is done asynchronously in application maintenance by the node repository) + // This call will remove application in zookeeper. Watches in TenantApplications will remove the application + // and allocated hosts in model and handlers in RPC server transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); @@ -766,7 +766,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return the active session, or null if there is no active session for the given application id. */ public RemoteSession getActiveSession(ApplicationId applicationId) { - return getActiveSession(getTenant(applicationId), applicationId); + Tenant tenant = getTenant(applicationId); + if (tenant == null) throw new IllegalArgumentException("Could not find any tenant for '" + applicationId + "'"); + return getActiveSession(tenant, applicationId); } public long getSessionIdForApplication(ApplicationId applicationId) { @@ -1057,12 +1059,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .getPort(); } - public Slime createDeployLog() { - Slime deployLog = new Slime(); - deployLog.setObject(); - return deployLog; - } - public Zone zone() { return new Zone(SystemName.from(configserverConfig.system()), Environment.from(configserverConfig.environment()), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 609ff4473c6..3275dc42477 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -57,7 +57,6 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final ApplicationRepository applicationRepository; private final RpcServer server; - private final Optional<Thread> serverThread; private final VersionState versionState; private final StateMonitor stateMonitor; private final VipStatus vipStatus; @@ -66,6 +65,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final Duration sleepTimeWhenRedeployingFails; private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails; private final ExecutorService rpcServerExecutor; + private final Optional<ExecutorService> bootstrapExecutor; @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, @@ -96,20 +96,21 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); + log.log(Level.FINE, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); initializing(vipStatusMode); switch (mode) { case BOOTSTRAP_IN_SEPARATE_THREAD: - this.serverThread = Optional.of(new Thread(this, "config server bootstrap thread")); - serverThread.get().start(); + bootstrapExecutor = Optional.of(Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server bootstrap"))); + bootstrapExecutor.get().execute(this); break; case BOOTSTRAP_IN_CONSTRUCTOR: - this.serverThread = Optional.empty(); + bootstrapExecutor = Optional.empty(); start(); break; case INITIALIZE_ONLY: - this.serverThread = Optional.empty(); + bootstrapExecutor = Optional.empty(); break; default: throw new IllegalArgumentException("Unknown bootstrap mode " + mode + ", legal values: " + Arrays.toString(Mode.values())); @@ -123,13 +124,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable server.stop(); log.log(Level.FINE, "RPC server stopped"); rpcServerExecutor.shutdown(); - serverThread.ifPresent(thread -> { - try { - thread.join(); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); - } - }); + bootstrapExecutor.ifPresent(ExecutorService::shutdownNow); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index 012d0d52275..ab1bd79d498 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -147,6 +147,11 @@ public class FileDistributionStatus extends AbstractComponent { } } + @Override + public void deconstruct() { + rpcExecutor.shutdownNow(); + } + static class HostStatus { private final String hostname; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java index d783dc105c3..dd9c8e4b6bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActions.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.yahoo.config.model.api.ConfigChangeAction; import java.util.List; +import java.util.Objects; /** * Contains an aggregated view of which actions that must be performed to handle config @@ -18,13 +19,16 @@ public class ConfigChangeActions { private final RefeedActions refeedActions; public ConfigChangeActions() { - this.restartActions = new RestartActions(); - this.refeedActions = new RefeedActions(); + this(new RestartActions(), new RefeedActions()); } public ConfigChangeActions(List<ConfigChangeAction> actions) { - this.restartActions = new RestartActions(actions); - this.refeedActions = new RefeedActions(actions); + this(new RestartActions(actions), new RefeedActions(actions)); + } + + public ConfigChangeActions(RestartActions restartActions, RefeedActions refeedActions) { + this.restartActions = Objects.requireNonNull(restartActions); + this.refeedActions = Objects.requireNonNull(refeedActions); } public RestartActions getRestartActions() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java index 29b0b99e42e..fab36246a41 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/configchange/RestartActions.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import java.util.*; +import java.util.stream.Collectors; /** * Represents all actions to restart services in order to handle a config change. @@ -18,6 +19,7 @@ public class RestartActions { private final String clusterName; private final String clusterType; private final String serviceType; + private final boolean ignoreForInternalRedeploy; private final Set<ServiceInfo> services = new LinkedHashSet<>(); private final Set<String> messages = new TreeSet<>(); @@ -31,10 +33,11 @@ public class RestartActions { return this; } - private Entry(String clusterName, String clusterType, String serviceType) { + private Entry(String clusterName, String clusterType, String serviceType, boolean ignoreForInternalRedeploy) { this.clusterName = clusterName; this.clusterType = clusterType; this.serviceType = serviceType; + this.ignoreForInternalRedeploy = ignoreForInternalRedeploy; } public String getClusterName() { @@ -49,6 +52,10 @@ public class RestartActions { return serviceType; } + public boolean ignoreForInternalRedeploy() { + return ignoreForInternalRedeploy; + } + public Set<ServiceInfo> getServices() { return services; } @@ -59,28 +66,19 @@ public class RestartActions { } - private Entry addEntry(ServiceInfo service) { - String clusterName = service.getProperty("clustername").orElse(""); - String clusterType = service.getProperty("clustertype").orElse(""); - String entryId = clusterType + "." + clusterName + "." + service.getServiceType(); - Entry entry = actions.get(entryId); - if (entry == null) { - entry = new Entry(clusterName, clusterType, service.getServiceType()); - actions.put(entryId, entry); - } - return entry; - } - private final Map<String, Entry> actions = new TreeMap<>(); - public RestartActions() { + public RestartActions() { } + + private RestartActions(Map<String, Entry> actions) { + this.actions.putAll(actions); } public RestartActions(List<ConfigChangeAction> actions) { for (ConfigChangeAction action : actions) { if (action.getType().equals(ConfigChangeAction.Type.RESTART)) { for (ServiceInfo service : action.getServices()) { - addEntry(service). + addEntry(service, action.ignoreForInternalRedeploy()). addService(service). addMessage(action.getMessage()); } @@ -88,6 +86,24 @@ public class RestartActions { } } + private Entry addEntry(ServiceInfo service, boolean ignoreForInternalRedeploy) { + String clusterName = service.getProperty("clustername").orElse(""); + String clusterType = service.getProperty("clustertype").orElse(""); + String entryId = clusterType + "." + clusterName + "." + service.getServiceType() + "." + ignoreForInternalRedeploy; + Entry entry = actions.get(entryId); + if (entry == null) { + entry = new Entry(clusterName, clusterType, service.getServiceType(), ignoreForInternalRedeploy); + actions.put(entryId, entry); + } + return entry; + } + + public RestartActions useForInternalRestart(boolean useForInternalRestart) { + return new RestartActions(actions.entrySet().stream() + .filter(entry -> !useForInternalRestart || !entry.getValue().ignoreForInternalRedeploy()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + public List<Entry> getEntries() { return new ArrayList<>(actions.values()); } @@ -97,6 +113,6 @@ public class RestartActions { } public boolean isEmpty() { - return getEntries().isEmpty(); + return actions.isEmpty(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index c33c5ff9f57..110c6464eba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -4,9 +4,11 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.util.logging.Level; @@ -22,25 +24,24 @@ public class DeployHandlerLogger implements DeployLogger { private static final Logger log = Logger.getLogger(DeployHandlerLogger.class.getName()); - private final Cursor logroot; + private final String prefix; private final boolean verbose; - private final ApplicationId app; + private final Slime slime; + private final Cursor logroot; - public DeployHandlerLogger(Cursor root, boolean verbose, ApplicationId app) { - logroot = root; + private DeployHandlerLogger(String prefix, boolean verbose) { + this.prefix = prefix; this.verbose = verbose; - this.app = app; + this.slime = new Slime(); + this.logroot = slime.setObject().setArray("log"); } @Override public void log(Level level, String message) { - if ((level == Level.FINE || - level == LogLevel.DEBUG || - level == LogLevel.SPAM) && - !verbose) { + if ((level == Level.FINE || level == LogLevel.DEBUG || level == LogLevel.SPAM) && !verbose) return; - } - String fullMsg = TenantRepository.logPre(app) + message; + + String fullMsg = prefix + message; Cursor entry = logroot.addObject(); entry.setLong("time", System.currentTimeMillis()); entry.setString("level", level.getName()); @@ -49,4 +50,19 @@ public class DeployHandlerLogger implements DeployLogger { log.log(Level.FINE, fullMsg); } + public Slime slime() { + return slime; + } + + public static DeployHandlerLogger forApplication(ApplicationId app, boolean verbose) { + return new DeployHandlerLogger(TenantRepository.logPre(app), verbose); + } + + public static DeployHandlerLogger forTenant(TenantName tenantName, boolean verbose) { + return new DeployHandlerLogger(TenantRepository.logPre(tenantName), verbose); + } + + public static DeployHandlerLogger forPrepareParams(PrepareParams prepareParams) { + return forApplication(prepareParams.getApplicationId(), prepareParams.isVerbose()); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 6c111ff0131..3726ea97fcc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -1,31 +1,34 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.yahoo.component.Version; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; -import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; @@ -44,82 +47,62 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** The session containing the application instance to activate */ private final LocalSession session; private final ApplicationRepository applicationRepository; - private final Optional<Provisioner> hostProvisioner; + private final Supplier<PrepareParams> params; + private final Optional<Provisioner> provisioner; private final Tenant tenant; - private final Duration timeout; + private final DeployLogger deployLogger; private final Clock clock; - private final DeployLogger logger = new SilentDeployLogger(); - - /** The repository part of docker image this application should run on. Version is separate from image repo */ - final Optional<DockerImage> dockerImageRepository; - - /** The Vespa version this application should run on */ - private final Version version; - - /** True if this deployment is done to bootstrap the config server */ - private final boolean isBootstrap; + private final boolean internalRedeploy; - /** The (optional) Athenz domain this application should use */ - private final Optional<AthenzDomain> athenzDomain; + private boolean prepared; + private ConfigChangeActions configChangeActions; - private boolean prepared = false; - - /** Whether this model should be validated (only takes effect if prepared=false) */ - private final boolean validate; - - /** Whether activation of this model should be forced */ - private final boolean force; - - private Deployment(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, - Clock clock, boolean prepared, boolean validate, boolean isBootstrap, boolean force) { + private Deployment(LocalSession session, ApplicationRepository applicationRepository, Supplier<PrepareParams> params, + Optional<Provisioner> provisioner, Tenant tenant, DeployLogger deployLogger, Clock clock, + boolean internalRedeploy, boolean prepared) { this.session = session; this.applicationRepository = applicationRepository; - this.hostProvisioner = hostProvisioner; + this.params = params; + this.provisioner = provisioner; this.tenant = tenant; - this.timeout = timeout; + this.deployLogger = deployLogger; this.clock = clock; + this.internalRedeploy = internalRedeploy; this.prepared = prepared; - this.validate = validate; - this.dockerImageRepository = session.getDockerImageRepository(); - this.version = session.getVespaVersion(); - this.isBootstrap = isBootstrap; - this.athenzDomain = session.getAthenzDomain(); - this.force = force; } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false, - validate, isBootstrap, false); + Optional<Provisioner> provisioner, Tenant tenant, PrepareParams params, DeployLogger logger, Clock clock) { + return new Deployment(session, applicationRepository, () -> params, provisioner, tenant, logger, clock, false, false); + } + + public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, + Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, + Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean internalRestart) { + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, internalRestart); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true, false); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, + Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, Duration timeout, Clock clock, boolean isBootstrap, boolean force) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, true, true, isBootstrap, force); + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, false, force, false); + return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, false, true); } /** Prepares this. This does nothing if this is already prepared */ @Override public void prepare() { if (prepared) return; - ApplicationId applicationId = session.getApplicationId(); - try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + PrepareParams params = this.params.get(); + if (params.internalRestart() && provisioner.isEmpty()) + throw new IllegalArgumentException("Internal restart not supported without Provisioner"); - PrepareParams.Builder params = new PrepareParams.Builder().applicationId(applicationId) - .timeoutBudget(timeoutBudget) - .ignoreValidationErrors(!validate) - .vespaVersion(version.toString()) - .isBootstrap(isBootstrap); - dockerImageRepository.ifPresent(params::dockerImageRepository); - athenzDomain.ifPresent(params::athenzDomain); + ApplicationId applicationId = params.getApplicationId(); + try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); - tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, - tenant.getPath(), clock.instant()); + this.configChangeActions = tenant.getSessionRepository().prepareLocalSession( + session, deployLogger, params, activeApplicationSet, tenant.getPath(), clock.instant()); this.prepared = true; } } @@ -130,16 +113,17 @@ public class Deployment implements com.yahoo.config.provision.Deployment { prepare(); validateSessionStatus(session); + PrepareParams params = this.params.get(); ApplicationId applicationId = session.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.activateMillis")) { - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + TimeoutBudget timeoutBudget = params.getTimeoutBudget(); if ( ! timeoutBudget.hasTimeLeft()) throw new RuntimeException("Timeout exceeded when trying to activate '" + applicationId + "'"); RemoteSession previousActiveSession; CompletionWaiter waiter; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { previousActiveSession = applicationRepository.getActiveSession(applicationId); - waiter = applicationRepository.activate(session, previousActiveSession, applicationId, force); + waiter = applicationRepository.activate(session, previousActiveSession, applicationId, params.force()); } catch (RuntimeException e) { throw e; @@ -150,10 +134,28 @@ public class Deployment implements com.yahoo.config.provision.Deployment { waiter.awaitCompletion(timeoutBudget.timeLeft()); log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + - hostProvisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + + provisioner.map(provisioner -> provisioner.getClass().getSimpleName()).orElse("no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + (previousActiveSession != null ? ". Based on session " + previousActiveSession.getSessionId() : "") + ". File references: " + applicationRepository.getFileReferences(applicationId)); + + if (params.internalRestart()) { + RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy); + + if (!restartActions.isEmpty()) { + Set<String> hostnames = restartActions.getEntries().stream() + .flatMap(entry -> entry.getServices().stream()) + .map(ServiceInfo::getHostName) + .collect(Collectors.toUnmodifiableSet()); + + provisioner.get().restart(applicationId, HostFilter.from(hostnames, Set.of(), Set.of(), Set.of())); + deployLogger.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", + hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); + + this.configChangeActions = new ConfigChangeActions(new RestartActions(), configChangeActions.getRefeedActions()); + } + } + return session.getMetaData().getGeneration(); } } @@ -165,12 +167,21 @@ public class Deployment implements com.yahoo.config.provision.Deployment { */ @Override public void restart(HostFilter filter) { - hostProvisioner.get().restart(session.getApplicationId(), filter); + provisioner.get().restart(session.getApplicationId(), filter); } /** Exposes the session of this for testing only */ public LocalSession session() { return session; } + /** + * @return config change actions that need to be performed as result of prepare + * @throws IllegalArgumentException if called without being prepared by this + */ + public ConfigChangeActions configChangeActions() { + if (configChangeActions != null) return configChangeActions; + throw new IllegalArgumentException("No config change actions: " + (prepared ? "was already prepared" : "not yet prepared")); + } + private void validateSessionStatus(LocalSession localSession) { long sessionId = localSession.getSessionId(); if (Session.Status.NEW.equals(localSession.getStatus())) { @@ -180,4 +191,36 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } + /** + * @param clock system clock + * @param timeout total timeout duration of prepare + activate + * @param session the local session for this deployment + * @param isBootstrap true if this deployment is done to bootstrap the config server + * @param ignoreValidationErrors whether this model should be validated + * @param force whether activation of this model should be forced + */ + private static Supplier<PrepareParams> createPrepareParams( + Clock clock, Duration timeout, LocalSession session, + boolean isBootstrap, boolean ignoreValidationErrors, boolean force, boolean internalRestart) { + + // Supplier because shouldn't/cant create this before validateSessionStatus() for prepared deployments + // memoized because we want to create this once for unprepared deployments + return Suppliers.memoize(() -> { + TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); + + PrepareParams.Builder params = new PrepareParams.Builder() + .applicationId(session.getApplicationId()) + .vespaVersion(session.getVespaVersion().toString()) + .timeoutBudget(timeoutBudget) + .ignoreValidationErrors(ignoreValidationErrors) + .isBootstrap(isBootstrap) + .force(force) + .internalRestart(internalRestart); + session.getDockerImageRepository().ifPresent(params::dockerImageRepository); + session.getAthenzDomain().ifPresent(params::athenzDomain); + + return params.build(); + }); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 87b0ed965d3..48d3fd6a176 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -164,7 +164,11 @@ public class ModelContextImpl implements ModelContext { private final Optional<AthenzDomain> athenzDomain; private final Optional<ApplicationRoles> applicationRoles; private final double feedCoreThreadPoolSizeFactor; + private final double visibilityDelay; private final Quota quota; + private final boolean tlsUseFSync; + private final String tlsCompressionType; + private final boolean useNewRestapiHandler; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -204,6 +208,12 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + visibilityDelay = Flags.VISIBILITY_DELAY.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + tlsCompressionType = Flags.TLS_COMPRESSION_TYPE.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + tlsUseFSync = Flags.TLS_USE_FSYNC.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) @@ -225,6 +235,9 @@ public class ModelContextImpl implements ModelContext { feedCoreThreadPoolSizeFactor = Flags.FEED_CORE_THREAD_POOL_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.quota = maybeQuota.orElseGet(Quota::empty); + this.useNewRestapiHandler = Flags.USE_NEW_RESTAPI_HANDLER.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()) + .value(); } @Override @@ -313,8 +326,12 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public double feedCoreThreadPoolSizeFactor() { return feedCoreThreadPoolSizeFactor; } - + @Override public double visibilityDelay() { return visibilityDelay; } + @Override public boolean tlsUseFSync() { return tlsUseFSync; } + @Override public String tlsCompressionType() { return tlsCompressionType; } @Override public Quota quota() { return quota; } + + @Override public boolean useNewRestapiHandler() { return useNewRestapiHandler; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 99cdb0a74dc..31294faad05 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; @@ -79,8 +80,10 @@ public class FileServer { private FileServer(ConnectionPool connectionPool, File rootDir) { this.downloader = new FileDownloader(connectionPool); this.root = new FileDirectory(rootDir); - this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); - this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors())); + this.pushExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("file server push")); + this.pullExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), + new DaemonThreadFactory("file server pull")); } boolean hasFile(String fileReference) { @@ -191,6 +194,8 @@ public class FileServer { public void close() { downloader.close(); + pullExecutor.shutdown(); + pushExecutor.shutdown(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java index 7112c7d3e23..bbbc8764122 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/ContentHandler.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; @@ -102,6 +103,6 @@ public class ContentHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); root.setString("prepared", request.getUrlBase("/prepared")); - return new SessionResponse(slime, root); + return new SlimeJsonResponse(slime); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java index f761f5f6b6e..57095772b26 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentListResponse.java @@ -2,12 +2,9 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.util.List; /** @@ -16,19 +13,13 @@ import java.util.List; * @author Ulf Lilleengen * @since 5.1 */ -class SessionContentListResponse extends SessionResponse { - private final Slime slime = new Slime(); +class SessionContentListResponse extends SlimeJsonResponse { public SessionContentListResponse(String urlBase, List<ApplicationFile> files) { - super(); Cursor array = slime.setArray(); for (ApplicationFile file : files) { array.addString(urlBase + file.getPath() + (file.isDirectory() ? "/" : "")); } } - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java index 075b4bc329b..08cda869111 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -3,12 +3,10 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import java.util.logging.Level; + +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import java.io.IOException; -import java.io.OutputStream; import java.util.*; /** @@ -16,14 +14,11 @@ import java.util.*; * * @author hmusum */ -class SessionContentStatusListResponse extends SessionResponse { +class SessionContentStatusListResponse extends SlimeJsonResponse { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger("SessionContentStatusListResponse"); - private final Slime slime = new Slime(); - public SessionContentStatusListResponse(String urlBase, List<ApplicationFile> files) { - super(); Cursor array = slime.setArray(); for (ApplicationFile f : files) { Cursor element = array.addObject(); @@ -34,9 +29,4 @@ class SessionContentStatusListResponse extends SessionResponse { } } - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java index bd182093e99..e6909e32985 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusResponse.java @@ -1,28 +1,19 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; -import java.io.*; /** * Represents a response for a request to show the status and md5sum of a file in the application package. * * @author hmusum */ -public class SessionContentStatusResponse extends SessionResponse { - - private final ApplicationFile file; - private final String urlBase; - private final ApplicationFile.MetaData metaData; - private final ObjectMapper mapper = new ObjectMapper(); +public class SessionContentStatusResponse extends SlimeJsonResponse { public SessionContentStatusResponse(ApplicationFile file, String urlBase) { - super(); - this.file = file; - this.urlBase = urlBase; - ApplicationFile.MetaData metaData; if (file == null) { metaData = new ApplicationFile.MetaData(ApplicationFile.ContentStatusDeleted, ""); @@ -32,24 +23,11 @@ public class SessionContentStatusResponse extends SessionResponse { if (metaData == null) { throw new IllegalArgumentException("Could not find status for '" + file.getPath() + "'"); } - this.metaData = metaData; - } - @Override - public void render(OutputStream outputStream) throws IOException { - mapper.writeValue(outputStream, new ResponseData(metaData.status, metaData.md5, urlBase + file.getPath())); - } - - private static class ResponseData { - public final String status; - public final String md5; - public final String name; - - private ResponseData(String status, String md5, String name) { - this.status = status; - this.md5 = md5; - this.name = name; - } + Cursor element = slime.setObject(); + element.setString("status", metaData.getStatus()); + element.setString("md5", metaData.getMd5()); + element.setString("name", urlBase + file.getPath()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java index 59d12e037e9..fcac023eec3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionHandler.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.TimeoutBudget; import java.time.Clock; @@ -67,14 +64,6 @@ public class SessionHandler extends HttpHandler { return new TimeoutBudget(Clock.systemUTC(), getRequestTimeout(request, defaultTimeout)); } - public static DeployHandlerLogger createLogger(Slime deployLog, HttpRequest request, ApplicationId app) { - return createLogger(deployLog, request.getBooleanProperty("verbose"), app); - } - - public static DeployHandlerLogger createLogger(Slime deployLog, boolean verbose, ApplicationId app) { - return new DeployHandlerLogger(deployLog.get().setArray("log"), verbose, app); - } - /** * True if this request should ignore activation failure because the session was made from an active session that is not active now * @param request a {@link com.yahoo.container.jdisc.HttpRequest} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java deleted file mode 100644 index ad658e3848a..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionResponse.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; - -import static com.yahoo.jdisc.http.HttpResponse.Status.OK; - -/** - * Superclass for responses from session HTTP handlers. Implements the - * render method. - * - * @author hmusum - * @since 5.1.14 - */ -public class SessionResponse extends HttpResponse { - private final Slime slime; - protected final Cursor root; - - public SessionResponse() { - super(OK); - slime = new Slime(); - root = slime.setObject(); - } - - public SessionResponse(Slime slime, Cursor root) { - super(OK); - this.slime = slime; - this.root = root; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - new JsonFormat(true).encode(outputStream, slime); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index d6badb8a9a2..9ea96b97af3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -13,11 +13,9 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.time.Duration; -import java.time.Instant; import static com.yahoo.vespa.config.server.application.CompressedApplicationInputStream.createFromCompressedStream; import static com.yahoo.vespa.config.server.http.Utils.checkThatTenantExists; @@ -56,7 +54,7 @@ public class ApplicationApiHandler extends SessionHandler { TenantName tenantName = validateTenant(request); PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); CompressedApplicationInputStream compressedStream = createFromCompressedStream(request.getData(), request.getHeader(contentTypeHeader)); - PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.deploy(compressedStream, prepareParams); return new SessionPrepareAndActivateResponse(result, request, prepareParams.getApplicationId(), zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java index 95a71881b47..3789939429c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListTenantsResponse.java @@ -3,26 +3,19 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Tenant list response * * @author vegardh */ -public class ListTenantsResponse extends SessionResponse { +public class ListTenantsResponse extends SlimeJsonResponse { ListTenantsResponse(ImmutableSet<TenantName> tenants) { - super(); - Cursor tenantArray = this.root.setArray("tenants"); + Cursor tenantArray = slime.setObject().setArray("tenants"); tenants.forEach(tenantName -> tenantArray.addString(tenantName.value())); } - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java index bb2b57ba45c..24bdfd81f1c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/PrepareResult.java @@ -1,8 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; /** * Encapsulates the result from preparing an application @@ -13,12 +13,12 @@ public class PrepareResult { private final long sessionId; private final ConfigChangeActions configChangeActions; - private final Slime deployLog; + private final DeployHandlerLogger logger; - public PrepareResult(long sessionId, ConfigChangeActions configChangeActions, Slime deployLog) { + public PrepareResult(long sessionId, ConfigChangeActions configChangeActions, DeployHandlerLogger logger) { this.sessionId = sessionId; this.configChangeActions = configChangeActions; - this.deployLog = deployLog; + this.logger = logger; } public long sessionId() { @@ -29,8 +29,8 @@ public class PrepareResult { return configChangeActions; } - public Slime deployLog() { - return deployLog; + public DeployHandlerLogger deployLogger() { + return logger; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java index 334dbe88614..9c0fbdf2613 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveResponse.java @@ -4,16 +4,19 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.http.SessionResponse; -public class SessionActiveResponse extends SessionResponse { +public class SessionActiveResponse extends SlimeJsonResponse { public SessionActiveResponse(Slime metaData, HttpRequest request, ApplicationId applicationId, long sessionId, Zone zone) { - super(metaData, metaData.get()); + super(metaData); TenantName tenantName = applicationId.tenant(); String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' activated."; + Cursor root = metaData.get(); + root.setString("tenant", tenantName.value()); root.setString("message", message); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java index 5aee711b379..b0468f5e608 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -11,7 +10,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.application.UriPattern; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -45,22 +43,25 @@ public class SessionCreateHandler extends SessionHandler { @Override protected HttpResponse handlePOST(HttpRequest request) { - Slime deployLog = applicationRepository.createDeployLog(); final TenantName tenantName = Utils.getTenantNameFromSessionRequest(request); Utils.checkThatTenantExists(applicationRepository.tenantRepository(), tenantName); TimeoutBudget timeoutBudget = SessionHandler.getTimeoutBudget(request, zookeeperBarrierTimeout); - DeployLogger logger = createLogger(request, deployLog, tenantName); + boolean verbose = request.getBooleanProperty("verbose"); + + DeployHandlerLogger logger; long sessionId; if (request.hasProperty("from")) { ApplicationId applicationId = getFromApplicationId(request); + logger = DeployHandlerLogger.forApplication(applicationId, verbose); sessionId = applicationRepository.createSessionFromExisting(applicationId, logger, false, timeoutBudget); } else { validateDataAndHeader(request); + logger = DeployHandlerLogger.forTenant(tenantName, verbose); // TODO: Avoid using application id here at all ApplicationId applicationId = ApplicationId.from(tenantName, ApplicationName.defaultName(), InstanceName.defaultName()); sessionId = applicationRepository.createSession(applicationId, timeoutBudget, request.getData(), request.getHeader(ApplicationApiHandler.contentTypeHeader)); } - return createResponse(request, tenantName, deployLog, sessionId); + return new SessionCreateResponse(logger.slime(), tenantName, request.getHost(), request.getPort(), sessionId); } static ApplicationId getFromApplicationId(HttpRequest request) { @@ -82,11 +83,6 @@ public class SessionCreateHandler extends SessionHandler { .instanceName(match.group(6)).build(); } - private static DeployHandlerLogger createLogger(HttpRequest request, Slime deployLog, TenantName tenant) { - return SessionHandler.createLogger(deployLog, request, - new ApplicationId.Builder().tenant(tenant).applicationName("-").build()); - } - static void validateDataAndHeader(HttpRequest request) { if (request.getData() == null) { throw new BadRequestException("Request contains no data"); @@ -99,9 +95,4 @@ public class SessionCreateHandler extends SessionHandler { ApplicationApiHandler.APPLICATION_X_GZIP + "' and '" + ApplicationApiHandler.APPLICATION_ZIP + "' are supported"); } } - - private HttpResponse createResponse(HttpRequest request, TenantName tenantName, Slime deployLog, long sessionId) { - return new SessionCreateResponse(tenantName, deployLog, deployLog.get()) - .createResponse(request.getHost(), request.getPort(), sessionId); - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java index 7d08ea94ce6..33c8f54b1f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionCreateResponse.java @@ -2,10 +2,9 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionCreateHandler. @@ -13,22 +12,17 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author hmusum * @since 5.1.27 */ -public class SessionCreateResponse extends SessionResponse { - private final TenantName tenantName; +public class SessionCreateResponse extends SlimeJsonResponse { - public SessionCreateResponse(TenantName tenantName, Slime deployLog, Cursor root) { - super(deployLog, root); - this.tenantName = tenantName; - } - - public HttpResponse createResponse(String hostName, int port, long sessionId) { + public SessionCreateResponse(Slime deployLog, TenantName tenantName, String hostName, int port, long sessionId) { + super(deployLog); String path = "http://" + hostName + ":" + port + "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId; + Cursor root = deployLog.get(); - this.root.setString("tenant", tenantName.value()); - this.root.setString("session-id", Long.toString(sessionId)); - this.root.setString("prepared", path + "/prepared"); - this.root.setString("content", path + "/content/"); - this.root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' created."); - return this; + root.setString("tenant", tenantName.value()); + root.setString("session-id", Long.toString(sessionId)); + root.setString("prepared", path + "/prepared"); + root.setString("content", path + "/content/"); + root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' created."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java index 7d9a0b11c28..7bace4749a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareAndActivateResponse.java @@ -5,21 +5,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareAndActivateResponse extends SessionResponse { +class SessionPrepareAndActivateResponse extends SlimeJsonResponse { SessionPrepareAndActivateResponse(PrepareResult result, HttpRequest request, ApplicationId applicationId, Zone zone) { - super(result.deployLog(), result.deployLog().get()); + super(result.deployLogger().slime()); + TenantName tenantName = applicationId.tenant(); String message = "Session " + result.sessionId() + " for tenant '" + tenantName.value() + "' prepared and activated."; - this.root.setString("tenant", tenantName.value()); + Cursor root = slime.get(); + + root.setString("tenant", tenantName.value()); root.setString("url", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName + "/application/" + applicationId.application().value() + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java index c0789a9c828..258af35be6f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandler.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.http.Utils; import java.time.Duration; -import java.time.Instant; /** * A handler that prepares a session given by an id in the request. v2 of application API @@ -41,7 +40,7 @@ public class SessionPrepareHandler extends SessionHandler { TenantName tenantName = tenant.getName(); long sessionId = getSessionIdV2(request); PrepareParams prepareParams = PrepareParams.fromHttpRequest(request, tenantName, zookeeperBarrierTimeout); - PrepareResult result = applicationRepository.prepare(tenant, sessionId, prepareParams, Instant.now()); + PrepareResult result = applicationRepository.prepare(tenant, sessionId, prepareParams); return new SessionPrepareResponse(result, tenantName, request); } @@ -51,7 +50,7 @@ public class SessionPrepareHandler extends SessionHandler { long sessionId = getSessionIdV2(request); applicationRepository.validateThatSessionIsNotActive(tenant, sessionId); applicationRepository.validateThatSessionIsPrepared(tenant, sessionId); - return new SessionPrepareResponse(applicationRepository.createDeployLog(), tenant.getName(), request, sessionId); + return new SessionPrepareResponse(tenant.getName(), request, sessionId); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java index 6d2aa426036..a97cd37d3b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareResponse.java @@ -3,33 +3,36 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActionsSlimeConverter; -import com.yahoo.vespa.config.server.http.SessionResponse; /** * Creates a response for SessionPrepareHandler. * * @author hmusum */ -class SessionPrepareResponse extends SessionResponse { +class SessionPrepareResponse extends SlimeJsonResponse { - SessionPrepareResponse(Slime deployLog, TenantName tenantName, HttpRequest request, long sessionId) { - this(deployLog, tenantName, request, sessionId, new ConfigChangeActions()); + SessionPrepareResponse(TenantName tenantName, HttpRequest request, long sessionId) { + this(new Slime(), tenantName, request, sessionId, new ConfigChangeActions()); } SessionPrepareResponse(PrepareResult result, TenantName tenantName, HttpRequest request) { - this(result.deployLog(), tenantName, request, result.sessionId(), result.configChangeActions()); + this(result.deployLogger().slime(), tenantName, request, result.sessionId(), result.configChangeActions()); } private SessionPrepareResponse(Slime deployLog, TenantName tenantName, HttpRequest request, long sessionId, ConfigChangeActions actions) { - super(deployLog, deployLog.get()); - String message = "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared."; - this.root.setString("tenant", tenantName.value()); - this.root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + + super(deployLog); + + Cursor root = deployLog.get().type() != Type.NIX ? deployLog.get() : deployLog.setObject(); + root.setString("tenant", tenantName.value()); + root.setString("activate", "http://" + request.getHost() + ":" + request.getPort() + "/application/v2/tenant/" + tenantName.value() + "/session/" + sessionId + "/active"); - root.setString("message", message); + root.setString("message", "Session " + sessionId + " for tenant '" + tenantName.value() + "' prepared."); new ConfigChangeActionsSlimeConverter(actions).toSlime(root); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java index 2850698ea87..6ff2b30075d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantCreateResponse.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant create @@ -11,16 +10,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author vegardh * */ -public class TenantCreateResponse extends SessionResponse { +public class TenantCreateResponse extends MessageResponse { public TenantCreateResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant "+tenant+" created."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant " + tenant + " created."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java index 3ba61f84270..d21584c8cdc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantDeleteResponse.java @@ -1,8 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant delete @@ -10,16 +9,10 @@ import com.yahoo.vespa.config.server.http.SessionResponse; * @author vegardh * */ -public class TenantDeleteResponse extends SessionResponse { +public class TenantDeleteResponse extends MessageResponse { public TenantDeleteResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant "+tenant+" deleted."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant " + tenant + " deleted."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java index 183fcd9b3c1..b918cab7828 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantGetResponse.java @@ -2,24 +2,17 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.HttpConfigResponse; -import com.yahoo.vespa.config.server.http.SessionResponse; +import com.yahoo.restapi.MessageResponse; /** * Response for tenant create * * @author hmusum */ -public class TenantGetResponse extends SessionResponse { +public class TenantGetResponse extends MessageResponse { public TenantGetResponse(TenantName tenant) { - super(); - this.root.setString("message", "Tenant '" + tenant + "' exists."); - } - - @Override - public String getContentType() { - return HttpConfigResponse.JSON_CONTENT_TYPE; + super("Tenant '" + tenant + "' exists."); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index 7ab55b9af72..9820eac2f30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -52,7 +52,9 @@ public class DelayedConfigResponses { // Since JRT does not allow adding watcher for "fake" requests, we must be able to disable it for unit tests :( DelayedConfigResponses(RpcServer rpcServer, int numTimerThreads, boolean useJrtWatcher) { this.rpcServer = rpcServer; - ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(numTimerThreads, ThreadFactoryFactory.getThreadFactory(DelayedConfigResponses.class.getName())); + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor(numTimerThreads, + ThreadFactoryFactory.getDaemonThreadFactory("delayed config responses")); executor.setRemoveOnCancelPolicy(true); this.executorService = executor; this.useJrtWatcher = useJrtWatcher; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index a96ae16a20b..55ee45482f9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -133,7 +133,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(config.maxgetconfigclients()); int rpcWorkerThreads = (config.numRpcThreads() == 0) ? threadsToUse() : config.numRpcThreads(); executorService = new ThreadPoolExecutor(rpcWorkerThreads, rpcWorkerThreads, - 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getThreadFactory(THREADPOOL_NAME)); + 0, TimeUnit.SECONDS, workQueue, ThreadFactoryFactory.getDaemonThreadFactory(THREADPOOL_NAME)); delayedConfigResponses = new DelayedConfigResponses(this, config.numDelayedResponseThreads()); spec = new Spec(null, config.rpcport()); hostRegistry = hostRegistries.getTenantHostRegistry(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index d962218b63a..914c927698f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -45,6 +45,7 @@ public final class PrepareParams { static final String APPLICATION_CONTAINER_ROLE = "applicationContainerRole"; static final String QUOTA_PARAM_NAME = "quota"; static final String FORCE_PARAM_NAME = "force"; + static final String INTERNAL_RESTART_PARAM_NAME = "internalRestart"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -53,6 +54,7 @@ public final class PrepareParams { private final boolean verbose; private final boolean isBootstrap; private final boolean force; + private final boolean internalRestart; private final Optional<Version> vespaVersion; private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; @@ -67,7 +69,8 @@ public final class PrepareParams { List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, - Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) { + Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force, + boolean internalRestart) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -83,6 +86,7 @@ public final class PrepareParams { this.applicationRoles = applicationRoles; this.quota = quota; this.force = force; + this.internalRestart = internalRestart; } public static class Builder { @@ -92,6 +96,7 @@ public final class PrepareParams { private boolean verbose = false; private boolean isBootstrap = false; private boolean force = false; + private boolean internalRestart = false; private ApplicationId applicationId = null; private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60)); private Optional<Version> vespaVersion = Optional.empty(); @@ -208,11 +213,16 @@ public final class PrepareParams { return this; } + public Builder internalRestart(boolean internalRestart) { + this.internalRestart = internalRestart; + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata, dockerImageRepository, athenzDomain, - applicationRoles, quota, force); + applicationRoles, quota, force, internalRestart); } } @@ -231,6 +241,7 @@ public final class PrepareParams { .applicationRoles(ApplicationRoles.fromString(request.getProperty(APPLICATION_HOST_ROLE), request.getProperty(APPLICATION_CONTAINER_ROLE))) .quota(request.getProperty(QUOTA_PARAM_NAME)) .force(request.getBooleanProperty(FORCE_PARAM_NAME)) + .internalRestart(request.getBooleanProperty(INTERNAL_RESTART_PARAM_NAME)) .build(); } @@ -282,6 +293,8 @@ public final class PrepareParams { public boolean force() { return force; } + public boolean internalRestart() { return internalRestart; } + public TimeoutBudget getTimeoutBudget() { return timeoutBudget; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 4f577d8f62c..78bbe8b5438 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -47,7 +47,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -123,10 +122,11 @@ public class SessionRepository { // ---------------- Local sessions ---------------------------------------------------------------- public synchronized void addLocalSession(LocalSession session) { - localSessionCache.put(session.getSessionId(), session); long sessionId = session.getSessionId(); - RemoteSession remoteSession = createRemoteSession(sessionId); - addSessionStateWatcher(sessionId, remoteSession); + localSessionCache.put(sessionId, session); + if (remoteSessionCache.get(sessionId) == null) { + createRemoteSession(sessionId); + } } public LocalSession getLocalSession(long sessionId) { @@ -265,11 +265,6 @@ public class SessionRepository { return getSessionList(curator.getChildren(sessionsPath)); } - public void addRemoteSession(RemoteSession session) { - remoteSessionCache.put(session.getSessionId(), session); - metrics.incAddedSessions(); - } - public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { int deleted = 0; for (long sessionId : getRemoteSessions()) { @@ -343,17 +338,20 @@ public class SessionRepository { * * @param sessionId session id for the new session */ - public void sessionAdded(long sessionId) { + public synchronized void sessionAdded(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); if (sessionZKClient.readStatus().equals(Session.Status.DELETE)) return; - log.log(Level.FINE, () -> "Adding remote session to SessionRepository: " + sessionId); - RemoteSession remoteSession = createRemoteSession(sessionId); - loadSessionIfActive(remoteSession); - addRemoteSession(remoteSession); + log.log(Level.FINE, () -> "Adding remote session " + sessionId); + RemoteSession session = createRemoteSession(sessionId); + if (session.getStatus() == Session.Status.NEW) { + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + } else { + log.log(Level.WARNING, () -> session.logPre() + "Session " + sessionId + " added, but with unexpected status " + session.getStatus()); + } if (distributeApplicationPackage()) createLocalSessionUsingDistributedApplicationPackage(sessionId); - addSessionStateWatcher(sessionId, remoteSession); } void activate(RemoteSession session) { @@ -373,14 +371,20 @@ public class SessionRepository { } public void delete(RemoteSession remoteSession) { - LocalSession localSession = getLocalSession(remoteSession.getSessionId()); - remoteSession.deactivate(); - if (localSession == null) { - // This change will be picked up by directoryCache in this class, which will do the rest of the cleanup - try (Lock lock = lock(remoteSession.getSessionId())) { - remoteSession.delete(); - } - } else { + // This change will be picked up by directoryCache in this class, which will do the rest of + // the cleanup on all config servers + long sessionId = remoteSession.getSessionId(); + try (Lock lock = lock(sessionId)) { + // TODO: Change log level to FINE when debugging is finished + log.log(Level.INFO, () -> remoteSession.logPre() + "Deactivating and deleting remote session " + sessionId); + remoteSession.deactivate(); + remoteSession.delete(); + remoteSessionCache.remove(sessionId); + } + LocalSession localSession = getLocalSession(sessionId); + if (localSession != null) { + // TODO: Change log level to FINE when debugging is finished + log.log(Level.INFO, () -> localSession.logPre() + "Deleting local session " + sessionId); deleteLocalSession(localSession); } } @@ -430,26 +434,16 @@ public class SessionRepository { log.log(Level.FINE, () -> "Got child event: " + event); switch (event.getType()) { case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; case CHILD_REMOVED: case CONNECTION_RECONNECTED: sessionsChanged(); break; + default: + break; } }); } - private void synchronizeOnNew(List<Long> sessionList) { - for (long sessionId : sessionList) { - RemoteSession session = remoteSessionCache.get(sessionId); - if (session == null) continue; // session might have been deleted after getting session list - log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - session.confirmUpload(); - } - } - /** * Creates a new deployment session from an application package. * @@ -463,9 +457,13 @@ public class SessionRepository { return create(applicationDirectory, applicationId, activeSessionId, false, timeoutBudget); } - public RemoteSession createRemoteSession(long sessionId) { + public synchronized RemoteSession createRemoteSession(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + RemoteSession session = new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + remoteSessionCache.put(sessionId, session); + loadSessionIfActive(session); + addSessionStateWatcher(sessionId, session); + return session; } private void ensureSessionPathDoesNotExist(long sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index c6c08beea17..d6d08aaac6c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -24,18 +24,18 @@ public class SessionStateWatcher { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); private final Curator.FileCache fileCache; - private final RemoteSession remoteSession; + private final RemoteSession session; private final MetricUpdater metrics; private final Executor zkWatcherExecutor; private final SessionRepository sessionRepository; SessionStateWatcher(Curator.FileCache fileCache, - RemoteSession remoteSession, + RemoteSession session, MetricUpdater metrics, Executor zkWatcherExecutor, SessionRepository sessionRepository) { this.fileCache = fileCache; - this.remoteSession = remoteSession; + this.session = session; this.metrics = metrics; this.fileCache.addListener(this::nodeChanged); this.fileCache.start(); @@ -44,24 +44,24 @@ public class SessionStateWatcher { } private void sessionStatusChanged(Status newStatus) { - long sessionId = remoteSession.getSessionId(); + long sessionId = session.getSessionId(); switch (newStatus) { case NEW: case NONE: break; case PREPARE: createLocalSession(sessionId); - sessionRepository.prepare(remoteSession); + sessionRepository.prepare(session); break; case ACTIVATE: createLocalSession(sessionId); - sessionRepository.activate(remoteSession); + sessionRepository.activate(session); break; case DEACTIVATE: - sessionRepository.deactivate(remoteSession); + sessionRepository.deactivate(session); break; case DELETE: - sessionRepository.delete(remoteSession); + sessionRepository.delete(session); break; default: throw new IllegalStateException("Unknown status " + newStatus); @@ -75,7 +75,7 @@ public class SessionStateWatcher { } public long getSessionId() { - return remoteSession.getSessionId(); + return session.getSessionId(); } public void close() { @@ -93,12 +93,12 @@ public class SessionStateWatcher { ChildData node = fileCache.getCurrentData(); if (node != null) { newStatus = Status.parse(Utf8.toString(node.getData())); - log.log(Level.FINE, remoteSession.logPre() + "Session change: Session " - + remoteSession.getSessionId() + " changed status to " + newStatus.name()); + log.log(Level.FINE, session.logPre() + "Session change: Session " + + session.getSessionId() + " changed status to " + newStatus.name()); sessionStatusChanged(newStatus); } } catch (Exception e) { - log.log(Level.WARNING, remoteSession.logPre() + "Error handling session change to " + + log.log(Level.WARNING, session.logPre() + "Error handling session change to " + newStatus.name() + " for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 41377bdf317..57e49fe365a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -84,7 +85,8 @@ public class TenantRepository { private final ExecutorService zkCacheExecutor; private final StripedExecutor<TenantName> zkWatcherExecutor; private final ExecutorService bootstrapExecutor; - private final ScheduledExecutorService checkForRemovedApplicationsService = new ScheduledThreadPoolExecutor(1); + private final ScheduledExecutorService checkForRemovedApplicationsService = + new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications")); private final Optional<Curator.DirectoryCache> directoryCache; /** @@ -96,7 +98,8 @@ public class TenantRepository { public TenantRepository(GlobalComponentRegistry componentRegistry) { this.componentRegistry = componentRegistry; ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); - this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); + this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(), + new DaemonThreadFactory("bootstrap tenants")); this.curator = componentRegistry.getCurator(); metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); this.tenantListeners.add(componentRegistry.getTenantListener()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java index 641aa31a6b7..b0bc8dc90dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/version/VersionState.java @@ -5,29 +5,44 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.util.Optional; /** - * Contains version information for this configserver. + * + * Contains version information for this configserver. Stored both in file system and in ZooKeeper (uses + * data in ZooKeeper if distributeApplicationPackage and data found in ZooKeeper) * * @author Ulf Lilleengen */ public class VersionState { + static final Path versionPath = Path.fromString("/config/v2/vespa_version"); + private final File versionFile; + private final Curator curator; + private final BooleanFlag distributeApplicationPackage; @Inject - public VersionState(ConfigserverConfig config) { - this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version")); + public VersionState(ConfigserverConfig config, Curator curator, FlagSource flagsource) { + this(new File(Defaults.getDefaults().underVespaHome(config.configServerDBDir()), "vespa_version"), curator, flagsource); } - public VersionState(File versionFile) { + public VersionState(File versionFile, Curator curator, FlagSource flagSource) { this.versionFile = versionFile; + this.curator = curator; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); } public boolean isUpgraded() { @@ -35,14 +50,29 @@ public class VersionState { } public void saveNewVersion() { + saveNewVersion(currentVersion().toFullString()); + } + + public void saveNewVersion(String vespaVersion) { + curator.set(versionPath, Utf8.toBytes(vespaVersion)); try (FileWriter writer = new FileWriter(versionFile)) { - writer.write(currentVersion().toFullString()); + writer.write(vespaVersion); } catch (IOException e) { throw new RuntimeException(e); } } public Version storedVersion() { + if (distributeApplicationPackage.value()) { + Optional<byte[]> version = curator.getData(versionPath); + if(version.isPresent()) { + try { + return Version.fromString(Utf8.toString(version.get())); + } catch (Exception e) { + // continue, use value in file + } + } + } try (FileReader reader = new FileReader(versionFile)) { return Version.fromString(IOUtils.readAll(reader)); } catch (Exception e) { @@ -54,6 +84,10 @@ public class VersionState { return new Version(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); } + File versionFile() { + return versionFile; + } + @Override public String toString() { return String.format("Current version:%s, stored version:%s", currentVersion(), storedVersion()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 4770b0797eb..e90ef38a92f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -382,8 +382,9 @@ public class ApplicationRepositoryTest { new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(serverdb.getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .sessionLifetime(60)); - DeployTester tester = new DeployTester(configserverConfig, clock); + DeployTester tester = new DeployTester.Builder().configserverConfig(configserverConfig).clock(clock).build(); tester.deployApp("src/test/apps/app", clock.instant()); // session 2 (numbering starts at 2) clock.advance(Duration.ofSeconds(10)); @@ -558,7 +559,7 @@ public class ApplicationRepositoryTest { long firstSession = result.sessionId(); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams()); exceptionRule.expect(RuntimeException.class); exceptionRule.expectMessage(containsString("Timeout exceeded when trying to activate 'test1.testapp'")); applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); @@ -583,7 +584,7 @@ public class ApplicationRepositoryTest { PrepareResult result2 = deployApp(testAppJdiscOnly); result2.sessionId(); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams()); exceptionRule.expect(ActivationConflictException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Cannot activate session 3 because the currently active session (4) has changed since session 3 was created (was 2 at creation time)")); applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId2, timeoutBudget, false); @@ -596,7 +597,7 @@ public class ApplicationRepositoryTest { exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("Session is active: 2")); - applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams()); exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Session 2 is already active")); @@ -704,7 +705,7 @@ public class ApplicationRepositoryTest { } private PrepareResult prepareAndActivate(File application) { - return applicationRepository.deploy(application, prepareParams(), Instant.now()); + return applicationRepository.deploy(application, prepareParams()); } private PrepareResult deployApp(File applicationPackage) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 23323d11f76..73f38188ec9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -25,6 +24,7 @@ import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -32,7 +32,6 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Arrays; @@ -56,6 +55,8 @@ import static org.junit.Assert.assertTrue; */ public class ConfigServerBootstrapTest { + private final MockCurator curator = new MockCurator(); + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -63,11 +64,11 @@ public class ConfigServerBootstrapTest { public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); @@ -96,11 +97,11 @@ public class ConfigServerBootstrapTest { public void testBootstrapWithVipStatusFile() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3", "host4"); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig, provisioner); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); @@ -121,11 +122,11 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWhenRedeploymentFails() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), configserverConfig); + DeployTester tester = new DeployTester.Builder().modelFactory(createHostedModelFactory()) + .configserverConfig(configserverConfig).build(); tester.deployApp("src/test/apps/hosted/"); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); // Manipulate application package so that it will fail deployment when config server starts @@ -158,18 +159,19 @@ public class ConfigServerBootstrapTest { public void testBootstrapNonHostedOneConfigModel() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfigNonHosted(temporaryFolder); String vespaVersion = "1.2.3"; - List<ModelFactory> modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString(vespaVersion))); List<Host> hosts = createHosts(vespaVersion); - InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); Curator curator = new MockCurator(); - DeployTester tester = new DeployTester(modelFactories, configserverConfig, - Clock.systemUTC(), new Zone(Environment.dev, RegionName.defaultName()), - provisioner, curator); + DeployTester tester = new DeployTester.Builder() + .modelFactory(DeployTester.createModelFactory(Version.fromString(vespaVersion))) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true)) + .configserverConfig(configserverConfig) + .zone(new Zone(Environment.dev, RegionName.defaultName())) + .curator(curator) + .build(); tester.deployApp("src/test/apps/app/", vespaVersion, Instant.now()); ApplicationId applicationId = tester.applicationId(); - File versionFile = temporaryFolder.newFile(); - VersionState versionState = new VersionState(versionFile); + VersionState versionState = createVersionState(); assertTrue(versionState.isUpgraded()); // Ugly hack, but I see no other way of doing it: @@ -220,6 +222,7 @@ public class ConfigServerBootstrapTest { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder("filedistribution").getAbsolutePath()) .hostedVespa(hosted) .multitenant(hosted) .maxDurationOfBootstrap(2) /* seconds */ @@ -241,6 +244,10 @@ public class ConfigServerBootstrapTest { stateMonitor); } + private VersionState createVersionState() throws IOException { + return new VersionState(temporaryFolder.newFile(), curator, new InMemoryFlagSource()); + } + public static class MockRpcServer extends com.yahoo.vespa.config.server.rpc.MockRpcServer { volatile boolean isRunning = false; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index e2c3369d49e..ead1e79a416 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.configchange; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import java.util.ArrayList; import java.util.List; @@ -22,11 +23,17 @@ public class ConfigChangeActionsBuilder { } public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName) { - actions.add(new MockRestartAction(message, - List.of(createService(clusterName, clusterType, serviceType, serviceName)))); + return restart(message, clusterName, clusterType, serviceType, serviceName, false); + } + + public ConfigChangeActionsBuilder restart(String message, String clusterName, String clusterType, String serviceType, String serviceName, boolean ignoreForInternalRedeploy) { + actions.add(new VespaRestartAction(message, + createService(clusterName, clusterType, serviceType, serviceName), + ignoreForInternalRedeploy)); return this; } + ConfigChangeActionsBuilder refeed(String name, boolean allowed, String message, String documentType, String clusterName, String serviceName) { actions.add(new MockRefeedAction(name, allowed, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java index bdf63befd15..904ca10aa1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRefeedAction.java @@ -30,6 +30,11 @@ public class MockRefeedAction extends MockConfigChangeAction implements ConfigCh public boolean allowed() { return allowed; } @Override + public boolean ignoreForInternalRedeploy() { + return false; + } + + @Override public String getDocumentType() { return documentType; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java deleted file mode 100644 index b1183f91282..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/MockRestartAction.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.configchange; - -import com.yahoo.config.model.api.ConfigChangeRestartAction; -import com.yahoo.config.model.api.ServiceInfo; - -import java.util.List; - -/** - * @author geirst - * @since 5.44 - */ -public class MockRestartAction extends MockConfigChangeAction implements ConfigChangeRestartAction { - public MockRestartAction(String message, List<ServiceInfo> services) { - super(message, services); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java index ee0180802af..c19b81aa91b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/RestartActionsTest.java @@ -5,8 +5,10 @@ import com.yahoo.config.model.api.ServiceInfo; import org.junit.Test; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -81,4 +83,18 @@ public class RestartActionsTest { assertThat(toString(entries.get(0)), equalTo("content.foo.searchnode:[baz][change]")); assertThat(toString(entries.get(1)), equalTo("search.foo.searchnode:[baz][change]")); } + + @Test + public void use_for_internal_restart_test() { + ConfigChangeActions actions = new ConfigChangeActionsBuilder() + .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE, SERVICE_TYPE, SERVICE_NAME) + .restart(CHANGE_MSG, CLUSTER, CLUSTER_TYPE_2, SERVICE_TYPE, SERVICE_NAME, true).build(); + + assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), + actions.getRestartActions().getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + assertEquals(Set.of(CLUSTER_TYPE, CLUSTER_TYPE_2), + actions.getRestartActions().useForInternalRestart(false).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + assertEquals(Set.of(CLUSTER_TYPE), + actions.getRestartActions().useForInternalRestart(true).getEntries().stream().map(RestartActions.Entry::getClusterType).collect(Collectors.toSet())); + } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index 6aa72e3e672..3ac9e681604 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -6,9 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import java.util.logging.Level; import com.yahoo.log.LogLevel; -import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; import org.junit.Test; @@ -33,13 +31,11 @@ public class DeployHandlerLoggerTest { } private void testLogging(boolean verbose, String expectedPattern) throws IOException { - Slime slime = new Slime(); - Cursor array = slime.setArray(); - DeployLogger logger = new DeployHandlerLogger(array, verbose, new ApplicationId.Builder() - .tenant("testtenant").applicationName("testapp").build()); + DeployHandlerLogger logger = DeployHandlerLogger.forApplication( + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build(), verbose); logMessages(logger); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - new JsonFormat(true).encode(baos, slime); + new JsonFormat(true).encode(baos, logger.slime()); assertTrue(Pattern.matches(expectedPattern, baos.toString())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index d92245bf5c1..7553583e70c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -42,6 +42,7 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; +import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.nio.file.Files; @@ -70,72 +71,16 @@ public class DeployTester { private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - public DeployTester() { - this(Collections.singletonList(createModelFactory(Clock.systemUTC()))); - } - - public DeployTester(List<ModelFactory> modelFactories) { - this(modelFactories, - new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) - .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString())), - Clock.systemUTC()); - } - - public DeployTester(ConfigserverConfig configserverConfig, Clock clock) { - this(Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { - this(modelFactories, configserverConfig, Clock.systemUTC()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { - this(modelFactories, configserverConfig, clock, Zone.defaultZone()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, HostProvisioner hostProvisioner) { - this(modelFactories, configserverConfig, Clock.systemUTC(), hostProvisioner); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { - this(modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { - this(modelFactories, configserverConfig, clock, zone, createProvisioner()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { - this(modelFactories, configserverConfig, clock, zone, provisioner, new MockCurator()); - } - - public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, - HostProvisioner provisioner, Curator curator) { + private DeployTester(Clock clock, TenantRepository tenantRepository, ApplicationRepository applicationRepository) { this.clock = clock; - TestComponentRegistry componentRegistry = createComponentRegistry(curator, Metrics.createTestMetrics(), - modelFactories, configserverConfig, clock, zone, - provisioner); - try { - this.tenantRepository = new TenantRepository(componentRegistry); - tenantRepository.addTenant(tenantName); - } - catch (Exception e) { - throw new IllegalArgumentException(e); - } - applicationRepository = new ApplicationRepository.Builder() - .withTenantRepository(tenantRepository) - .withProvisioner(new ProvisionerAdapter(provisioner)) - .withConfigserverConfig(configserverConfig) - .withOrchestrator(new OrchestratorMock()) - .withClock(clock) - .build(); + this.tenantRepository = tenantRepository; + this.applicationRepository = applicationRepository; } public Tenant tenant() { return tenantRepository.getTenant(tenantName); } - + /** Create a model factory for the version of this source*/ public static CountingModelFactory createModelFactory(Clock clock) { return new CountingModelFactory(clock); @@ -215,7 +160,7 @@ public class DeployTester { paramsBuilder.applicationId(applicationId) .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); - return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), now); + return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build()); } public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { @@ -296,11 +241,11 @@ public class DeployTester { private static class FailingModelFactory implements ModelFactory { private final Version version; - + public FailingModelFactory(Version version) { this.version = version; } - + @Override public Version version() { return version; } @@ -369,4 +314,99 @@ public class DeployTester { } + public static class Builder { + private Clock clock; + private Provisioner provisioner; + private ConfigserverConfig configserverConfig; + private Zone zone; + private Curator curator; + private Metrics metrics; + private List<ModelFactory> modelFactories; + private Orchestrator orchestrator; + + public DeployTester build() { + Clock clock = Optional.ofNullable(this.clock).orElseGet(Clock::systemUTC); + Zone zone = Optional.ofNullable(this.zone).orElseGet(Zone::defaultZone); + ConfigserverConfig configserverConfig = Optional.ofNullable(this.configserverConfig) + .orElseGet(() -> new ConfigserverConfig(new ConfigserverConfig.Builder() + .configServerDBDir(uncheck(() -> Files.createTempDirectory("serverdb")).toString()) + .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) + .fileReferencesDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()))); + Provisioner provisioner = Optional.ofNullable(this.provisioner) + .orElseGet(() -> new ProvisionerAdapter(createProvisioner())); + List<ModelFactory> modelFactories = Optional.ofNullable(this.modelFactories) + .orElseGet(() -> List.of(createModelFactory(clock))); + + TestComponentRegistry.Builder testComponentRegistryBuilder = new TestComponentRegistry.Builder() + .clock(clock) + .configServerConfig(configserverConfig) + .curator(Optional.ofNullable(curator).orElseGet(MockCurator::new)) + .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .metrics(Optional.ofNullable(metrics).orElseGet(Metrics::createTestMetrics)) + .zone(zone); + if (configserverConfig.hostedVespa()) testComponentRegistryBuilder.provisioner(provisioner); + + TenantRepository tenantRepository = new TenantRepository(testComponentRegistryBuilder.build()); + tenantRepository.addTenant(tenantName); + + ApplicationRepository applicationRepository = new ApplicationRepository.Builder() + .withTenantRepository(tenantRepository) + .withConfigserverConfig(configserverConfig) + .withOrchestrator(Optional.ofNullable(orchestrator).orElseGet(OrchestratorMock::new)) + .withClock(clock) + .withProvisioner(provisioner) + .build(); + + return new DeployTester(clock, tenantRepository, applicationRepository); + } + + public Builder clock(Clock clock) { + this.clock = clock; + return this; + } + + public Builder provisioner(Provisioner provisioner) { + this.provisioner = provisioner; + return this; + } + + public Builder hostProvisioner(HostProvisioner hostProvisioner) { + return provisioner(new ProvisionerAdapter(hostProvisioner)); + } + + public Builder configserverConfig(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public Builder curator(Curator curator) { + this.curator = curator; + return this; + } + + public Builder metrics(Metrics metrics) { + this.metrics = metrics; + return this; + } + + public Builder modelFactory(ModelFactory modelFactory) { + return modelFactories(List.of(modelFactory)); + } + + public Builder modelFactories(List<ModelFactory> modelFactories) { + this.modelFactories = modelFactories; + return this; + } + + public Builder orchestrator(Orchestrator orchestrator) { + this.orchestrator = orchestrator; + return this; + } + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 254fe62cba8..18ca54e8c11 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -20,12 +20,12 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.config.server.configchange.MockRestartAction; import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -65,8 +65,9 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() throws IOException { - CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/hosted/", "4.5.6"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); @@ -77,7 +78,9 @@ public class HostedDeployTest { @Test public void testRedeploy() throws IOException { - DeployTester tester = new DeployTester(List.of(createHostedModelFactory()), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory()) + .configserverConfig(createConfigserverConfig()).build(); ApplicationId appId = tester.applicationId(); tester.deployApp("src/test/apps/hosted/"); assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); @@ -90,8 +93,9 @@ public class HostedDeployTest { @Test public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException { - CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder() + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .configserverConfig(createConfigserverConfig()).build(); String dockerImageRepository = "docker.foo.com:4443/bar/baz"; tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder() .vespaVersion("4.5.6") @@ -111,7 +115,7 @@ public class HostedDeployTest { List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")), createHostedModelFactory(Version.fromString("6.2.0")), createHostedModelFactory(Version.fromString("7.0.0"))); - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/hosted/", "6.2.0"); assertEquals(4, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } @@ -332,7 +336,7 @@ public class HostedDeployTest { ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List<ModelFactory> modelFactories = List.of(createHostedModelFactory(clock), createFailingModelFactory(Version.fromString("1.0.0"))); // older than default - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).configserverConfig(createConfigserverConfig()).build(); tester.deployApp("src/test/apps/validationOverride/", clock.instant()); // Redeployment from local active works @@ -373,8 +377,8 @@ public class HostedDeployTest { new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); List<ModelFactory> modelFactories = List.of( - new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new MockRestartAction("change", services)), - new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new MockRestartAction("other change", services))); + new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), new VespaRestartAction("change", services)), + new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), new VespaRestartAction("other change", services))); DeployTester tester = createTester(hosts, modelFactories, prodZone); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.2.0"); @@ -393,6 +397,7 @@ public class HostedDeployTest { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .hostedVespa(true) .multitenant(true) .region(zone.region().value()) @@ -415,8 +420,12 @@ public class HostedDeployTest { private DeployTester createTester(List<Host> hosts, List<ModelFactory> modelFactories, Zone prodZone, Clock clock) throws IOException { - return new DeployTester(modelFactories, createConfigserverConfig(prodZone), - clock, prodZone, new InMemoryProvisioner(new Hosts(hosts), true)); + return new DeployTester.Builder() + .modelFactories(modelFactories) + .configserverConfig(createConfigserverConfig(prodZone)) + .clock(clock) + .zone(prodZone) + .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true)).build(); } private static class ConfigChangeActionsModelFactory extends TestModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index c07c7316930..015cc039a1c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -27,7 +27,7 @@ public class RedeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester(); + DeployTester tester = new DeployTester.Builder().build(); tester.deployApp("src/test/apps/app"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); @@ -45,7 +45,7 @@ public class RedeployTest { public void testNoRedeploy() { List<ModelFactory> modelFactories = List.of(createModelFactory(Clock.systemUTC()), createFailingModelFactory(Version.fromString("1.0.0"))); - DeployTester tester = new DeployTester(modelFactories); + DeployTester tester = new DeployTester.Builder().modelFactories(modelFactories).build(); ApplicationId id = ApplicationId.from(tester.tenant().getName(), ApplicationName.from("default"), InstanceName.from("default")); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 40aee72e71c..ea896469f03 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -48,12 +48,14 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -61,6 +63,7 @@ public class HttpGetConfigHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 906716125c2..9cf8f7960a3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -53,12 +53,14 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -66,6 +68,7 @@ public class HttpListConfigsHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 67ac0b02133..ab5303b221e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -17,7 +18,9 @@ import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -35,8 +38,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private static final File testApp = new File("src/test/apps/content"); private static final File testApp2 = new File("src/test/apps/content2"); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - private final Clock clock = componentRegistry.getClock(); + private final TenantName tenantName1 = TenantName.from("mofet"); private final TenantName tenantName2 = TenantName.from("bla"); @@ -48,8 +50,22 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private ApplicationRepository applicationRepository; private ApplicationHandler handler; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupHandler() { + public void setupHandler() throws IOException { + + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + Clock clock = componentRegistry.getClock(); + TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName1); tenantRepository.addTenant(tenantName2); @@ -59,6 +75,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams(appId1)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index d84b81bd8e7..9e4ca8dfb2c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; @@ -31,7 +32,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import javax.ws.rs.client.Client; import java.io.ByteArrayInputStream; @@ -73,12 +76,21 @@ public class ApplicationHandlerTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { List<ModelFactory> modelFactories = List.of(DeployTester.createModelFactory(vespaVersion)); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .provisioner(provisioner) .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) + .configServerConfig(configserverConfig) .build(); tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenantName); @@ -91,6 +103,7 @@ public class ApplicationHandlerTest { .withClock(componentRegistry.getClock()) .withTesterClient(testerClient) .withLogRetriever(logRetriever) + .withConfigserverConfig(configserverConfig) .build(); } @@ -357,7 +370,7 @@ public class ApplicationHandlerTest { assertEquals(200, response.getStatus()); String renderedString = SessionHandlerTest.getRenderedString(response); assertEquals("{\"generation\":" + expectedGeneration + - ",\"applicationPackageFileReference\":\"\"" + + ",\"applicationPackageFileReference\":\"./\"" + ",\"modelVersions\":[\"" + expectedVersion.toFullString() + "\"]}", renderedString); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 4d64721f7dd..6b9abf5d7ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,7 +20,9 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; @@ -40,10 +43,19 @@ public class HostHandlerTest { private final static Zone zone = Zone.defaultZone(); private ApplicationRepository applicationRepository; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .zone(zone) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenant); @@ -51,6 +63,7 @@ public class HostHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 14d7a0743a5..9bb113875b4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -57,12 +57,14 @@ public class HttpGetConfigHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -70,6 +72,7 @@ public class HttpGetConfigHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new HttpGetConfigHandler(HttpGetConfigHandler.testOnlyContext(), tenantRepository); applicationRepository.deploy(testApp, prepareParams()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 785c9977fd2..c1adec3336d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -61,12 +61,14 @@ public class HttpListConfigsHandlerTest { @Before public void setUp() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); @@ -74,6 +76,7 @@ public class HttpListConfigsHandlerTest { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, prepareParams()); handler = new HttpListConfigsHandler(HttpListConfigsHandler.testOnlyContext(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index c3a7e82dff5..511717acfc0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.provision.ApplicationId; @@ -65,12 +66,18 @@ public class SessionActiveHandlerTest { public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setup() { + public void setup() throws IOException { VespaModelFactory modelFactory = new TestModelFactory(Version.fromString("7.222.2")); hostProvisioner = new SessionHandlerTest.MockProvisioner(); + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); componentRegistry = new TestComponentRegistry.Builder() .curator(new MockCurator()) .modelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) + .configServerConfig(configserverConfig) .build(); TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); @@ -79,6 +86,7 @@ public class SessionActiveHandlerTest { .withProvisioner(hostProvisioner) .withOrchestrator(new OrchestratorMock()) .withClock(componentRegistry.getClock()) + .withConfigserverConfig(configserverConfig) .build(); handler = createHandler(); } @@ -133,8 +141,7 @@ public class SessionActiveHandlerTest { testApp); applicationRepository.prepare(tenant, sessionId, - new PrepareParams.Builder().applicationId(applicationId()).build(), - componentRegistry.getClock().instant()); + new PrepareParams.Builder().applicationId(applicationId()).build()); actResponse = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); LocalSession session = applicationRepository.getActiveLocalSession(tenant, applicationId()); metaData = session.getMetaData(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index d28404d8d72..6de85f12765 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -19,7 +20,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayInputStream; import java.io.File; @@ -37,14 +40,25 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { private static final TenantName tenantName = TenantName.from("contenttest"); private static final File testApp = new File("src/test/apps/content"); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - + private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; private SessionContentHandler handler = null; private long sessionId; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupHandler() { + public void setupHandler() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + componentRegistry = new TestComponentRegistry.Builder() + .configServerConfig(configserverConfig) + .build(); + tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); @@ -52,6 +66,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId()).build()); Tenant tenant = applicationRepository.getTenant(applicationId()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index cc4f39b0789..f1abddba63c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -25,7 +25,9 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; @@ -54,17 +56,30 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { private static final File app = new File("src/test/resources/deploy/validapp"); private final Curator curator = new MockCurator(); - private final TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - private final Clock clock = componentRegistry.getClock(); - private final TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); + private TimeoutBudget timeoutBudget; private ApplicationRepository applicationRepository; + private TestComponentRegistry componentRegistry; private String preparedMessage = " prepared.\"}"; private String tenantMessage = ""; private TenantRepository tenantRepository; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setupRepo() { + public void setupRepo() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(configserverConfig) + .build(); + Clock clock = componentRegistry.getClock(); + timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(10)); tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenant); applicationRepository = new ApplicationRepository.Builder() @@ -72,6 +87,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; preparedMessage = " for tenant '" + tenant + "' prepared.\""; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index ecab121e547..748c43bafeb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -8,29 +8,31 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.Clock; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.SessionResponse; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.NotFoundException; +import org.junit.rules.TemporaryFolder; public class TenantHandlerTest { @@ -41,13 +43,26 @@ public class TenantHandlerTest { private TenantHandler handler; private final TenantName a = TenantName.from("a"); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Before - public void setup() { - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder().curator(new MockCurator()).build()); + public void setup() throws IOException { + ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .build(); + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() + .curator(new MockCurator()) + .configServerConfig(configserverConfig) + .build()); + applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withProvisioner(new SessionHandlerTest.MockProvisioner()) .withOrchestrator(new OrchestratorMock()) + .withConfigserverConfig(configserverConfig) .build(); handler = new TenantHandler(TenantHandler.testOnlyContext(), applicationRepository); } @@ -141,7 +156,7 @@ public class TenantHandlerTest { return (TenantCreateResponse) handler.handlePUT(testRequest); } - private void assertResponseEquals(SessionResponse response, String payload) throws IOException { + private void assertResponseEquals(HttpResponse response, String payload) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); assertEquals(baos.toString(StandardCharsets.UTF_8), payload); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index 78d69b75d59..712242a69e6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -36,10 +36,8 @@ class MaintainerTester { private final Curator curator; private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - private final Clock clock; MaintainerTester(Clock clock, TemporaryFolder temporaryFolder) throws IOException { - this.clock = clock; this.curator = new MockCurator(); InMemoryProvisioner hostProvisioner = new InMemoryProvisioner(true, "host0", "host1", "host2", "host3", "host4"); ProvisionerAdapter provisioner = new ProvisionerAdapter(hostProvisioner); @@ -47,6 +45,7 @@ class MaintainerTester { .hostedVespa(true) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) @@ -67,7 +66,7 @@ class MaintainerTester { } void deployApp(File applicationPath, PrepareParams.Builder prepareParams) { - applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build(), clock.instant()); + applicationRepository.deploy(applicationPath, prepareParams.ignoreValidationErrors(true).build()); } Curator curator() { return curator; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index eb06f2f7017..47217491e3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -82,7 +82,8 @@ public class RpcTester implements AutoCloseable { spec = createSpec(port); configBuilder.rpcport(port) .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()); + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()); configserverConfig = new ConfigserverConfig(configBuilder); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .configDefinitionRepo(new TestConfigDefinitionRepo()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java deleted file mode 100644 index fb268492dd7..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.Version; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.time.Instant; -import java.util.Collections; -import java.util.Optional; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class LocalSessionTest { - - private static final File testApp = new File("src/test/apps/app"); - private static final TenantName tenantName = TenantName.from("test_tenant"); - private static final Path tenantPath = Path.createRoot(); - - private TenantRepository tenantRepository; - private Curator curator; - private ConfigCurator configCurator; - - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setupTest() throws IOException { - curator = new MockCurator(); - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .configServerConfig(new ConfigserverConfig.Builder() - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) - .build()) - .build(); - tenantRepository = new TenantRepository(componentRegistry); - tenantRepository.addTenant(tenantName); - configCurator = ConfigCurator.create(curator); - } - - @Test - public void require_that_session_is_initialized() throws Exception { - LocalSession session = createSession(applicationId(), 2); - assertThat(session.getSessionId(), is(2L)); - session = createSession(applicationId(), Long.MAX_VALUE); - assertThat(session.getSessionId(), is(Long.MAX_VALUE)); - assertThat(session.getActiveSessionAtCreate(), is(0L)); - } - - @Test - public void require_that_marking_session_modified_changes_status_to_new() throws Exception { - LocalSession session = createSession(applicationId(), 3); - doPrepare(session, applicationId()); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), Session.Mode.READ); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), Session.Mode.WRITE); - assertThat(session.getStatus(), is(Session.Status.NEW)); - } - - @Test - public void require_that_application_file_can_be_fetched() throws Exception { - LocalSession session = createSession(applicationId(), 3); - ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), Session.Mode.READ); - ApplicationFile f2 = session.getApplicationFile(Path.fromString("services2.xml"), Session.Mode.READ); - assertTrue(f1.exists()); - assertFalse(f2.exists()); - } - - @Test(expected = IllegalStateException.class) - public void require_that_no_provision_info_throws_exception() throws Exception { - createSession(applicationId(), 3).getAllocatedHosts(); - } - - private LocalSession createSession(ApplicationId applicationId, long sessionId) throws Exception { - return createSession(applicationId, sessionId, Optional.empty()); - } - - private LocalSession createSession(ApplicationId applicationId, long sessionId, - Optional<AllocatedHosts> allocatedHosts) throws Exception { - TenantName tenantName = applicationId.tenant(); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId, allocatedHosts); - zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); - ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), - TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); - if (allocatedHosts.isPresent()) { - zkClient.write(allocatedHosts.get()); - } - zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); - TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); - applications.createApplication(applicationId); - LocalSession session = new LocalSession(tenantName, sessionId, FilesApplicationPackage.fromFile(testApp), zkc); - session.setApplicationId(applicationId); - return session; - } - - private void doPrepare(LocalSession session, ApplicationId applicationId) { - doPrepare(session, new PrepareParams.Builder().applicationId(applicationId).build()); - } - - private void doPrepare(LocalSession session, PrepareParams params) { - SessionRepository sessionRepository = tenantRepository.getTenant(params.getApplicationId().tenant()).getSessionRepository(); - sessionRepository.prepareLocalSession(session, getLogger(), params, Optional.empty(), tenantPath, Instant.now()); - } - - private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false, applicationId()); - } - - private ApplicationId applicationId() { - return new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build(); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index b181ad3e8d6..95cdccf4cb8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -27,7 +28,6 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.TestComponentRegistry; @@ -65,6 +65,7 @@ import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -181,7 +182,10 @@ public class SessionPreparerTest { .dryRun(true) .build(), 1); - assertTrue(result.getFileRegistries().get(version321).export().isEmpty()); + Map<Version, FileRegistry> fileRegistries = result.getFileRegistries(); + System.out.println(fileRegistries); + assertEquals(1, fileRegistries.get(version321).export().size()); + assertEquals("./", fileRegistries.get(version321).export().get(0).reference.value()); } @Test @@ -383,8 +387,8 @@ public class SessionPreparerTest { } private DeployHandlerLogger getLogger() { - return new DeployHandlerLogger(new Slime().get(), false /*verbose */, - new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + return DeployHandlerLogger.forApplication( + new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build(), false /*verbose */); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java index 8a1a7fbea20..1340935108b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/version/VersionStateTest.java @@ -4,12 +4,16 @@ package com.yahoo.vespa.config.server.version; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; @@ -20,28 +24,49 @@ import static org.junit.Assert.assertTrue; * @author Ulf Lilleengen */ public class VersionStateTest { + InMemoryFlagSource flagSource = new InMemoryFlagSource(); @Rule public TemporaryFolder tempDir = new TemporaryFolder(); + private final MockCurator curator = new MockCurator(); @Test public void upgrade() throws IOException { + upgrade(true); + upgrade(false); + } + + public void upgrade(boolean distributeApplicationPackage) throws IOException { + flagSource.withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), distributeApplicationPackage); Version unknownVersion = new Version(0, 0, 0); - File versionFile = tempDir.newFile(); - VersionState state = new VersionState(versionFile); + + VersionState state = createVersionState(); assertThat(state.storedVersion(), is(unknownVersion)); assertTrue(state.isUpgraded()); state.saveNewVersion(); assertFalse(state.isUpgraded()); - IOUtils.writeFile(versionFile, "badversion", false); + state.saveNewVersion("badversion"); assertThat(state.storedVersion(), is(unknownVersion)); assertTrue(state.isUpgraded()); - IOUtils.writeFile(versionFile, "5.0.0", false); + state.saveNewVersion("5.0.0"); assertThat(state.storedVersion(), is(new Version(5, 0, 0))); assertTrue(state.isUpgraded()); + // Remove zk node, should find version in ZooKeeper + curator.delete(VersionState.versionPath); + assertThat(state.storedVersion(), is(new Version(5, 0, 0))); + assertTrue(state.isUpgraded()); + + // Save new version, remove version in file, should find version in ZooKeeper + state.saveNewVersion("6.0.0"); + if (distributeApplicationPackage) { + Files.delete(state.versionFile().toPath()); + assertThat(state.storedVersion(), is(new Version(6, 0, 0))); + assertTrue(state.isUpgraded()); + } + state.saveNewVersion(); assertThat(state.currentVersion(), is(state.storedVersion())); assertFalse(state.isUpgraded()); @@ -50,7 +75,9 @@ public class VersionStateTest { @Test public void serverdbfile() throws IOException { File dbDir = tempDir.newFolder(); - VersionState state = new VersionState(new ConfigserverConfig(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()))); + VersionState state = new VersionState(new ConfigserverConfig.Builder().configServerDBDir(dbDir.getAbsolutePath()).build(), + curator, + new InMemoryFlagSource()); state.saveNewVersion(); File versionFile = new File(dbDir, "vespa_version"); assertTrue(versionFile.exists()); @@ -58,4 +85,8 @@ public class VersionStateTest { assertThat(stored, is(state.currentVersion())); } + private VersionState createVersionState() throws IOException { + return new VersionState(tempDir.newFile(), curator, flagSource); + } + } |