summaryrefslogtreecommitdiffstats
path: root/configserver
diff options
context:
space:
mode:
authorHÃ¥kon Hallingstad <hakon@verizonmedia.com>2020-04-20 10:36:33 +0200
committerGitHub <noreply@github.com>2020-04-20 10:36:33 +0200
commitb105eead1fbbcefbb85bc962749f2a12fa660bbe (patch)
tree7a3c996c00b854066d32608a002335715fb98c96 /configserver
parentf61f6c701dc91e839b865f158a6da56ff166def7 (diff)
parent9ab0ef70e9ed4f422df67603f26bcb0c7918fdc4 (diff)
Merge branch 'master' into hakonhall/remove-use-bucket-space-metric-feature-flag
Diffstat (limited to 'configserver')
-rw-r--r--configserver/pom.xml5
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java12
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java17
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java34
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java9
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java58
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java48
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java83
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java3
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java8
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java22
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java29
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java12
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java29
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java21
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java45
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java24
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java17
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java9
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java13
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java42
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java1
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java6
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java27
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java44
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java13
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java22
-rw-r--r--configserver/src/main/resources/configserver-app/services.xml3
-rwxr-xr-xconfigserver/src/main/sh/start-configserver1
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java9
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java74
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java18
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java7
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java6
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java30
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java2
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java24
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java32
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java19
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java11
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java30
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java35
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java80
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java58
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java68
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java10
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java22
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java4
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java21
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java6
-rw-r--r--configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java10
60 files changed, 766 insertions, 484 deletions
diff --git a/configserver/pom.xml b/configserver/pom.xml
index 1e242988327..8cd1b4b4254 100644
--- a/configserver/pom.xml
+++ b/configserver/pom.xml
@@ -215,11 +215,6 @@
<artifactId>jersey-proxy-client</artifactId>
</dependency>
<dependency>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 92e7bf0300b..95c8733b540 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -314,8 +314,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
tenant.getLocalSessionRepo().addSession(newSession);
return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock,
- false /* don't validate as this is already deployed */, newSession.getVespaVersion(),
- bootstrap));
+ false /* don't validate as this is already deployed */, bootstrap));
}
@Override
@@ -893,11 +892,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
public void close() {
metric.set(name,
Duration.between(start, clock.instant()).toMillis(),
- metric.createContext(Map.of("tenant", id.tenant().value(),
- "application", id.application().value(),
- "instance", id.instance().value(),
- "environment", environment,
- "region", region)));
+ metric.createContext(Map.of("applicationId", id.toFullString(),
+ "tenantName", id.tenant().value(),
+ "app", id.application().value() + "." + id.instance().value(),
+ "zone", environment + "." + region)));
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index b2a10f4bb21..2a426e4cccc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -79,7 +79,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
// For testing only
ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
- StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) {
+ StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) {
this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
}
@@ -231,20 +231,17 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
new DaemonThreadFactory("redeploy apps"));
// Keep track of deployment per application
Map<ApplicationId, Future<?>> futures = new HashMap<>();
- Set<ApplicationId> failedDeployments = new HashSet<>();
-
- for (ApplicationId appId : applicationIds) {
- Optional<Deployment> deploymentOptional = applicationRepository.deployFromLocalActive(appId, true /* bootstrap */);
- if (deploymentOptional.isEmpty()) continue;
-
- futures.put(appId, executor.submit(deploymentOptional.get()::activate));
- }
+ applicationIds.forEach(applicationId -> futures.put(applicationId, executor.submit(() -> {
+ applicationRepository.deployFromLocalActive(applicationId, true /* bootstrap */)
+ .ifPresent(Deployment::activate);
+ })));
+ Set<ApplicationId> failedDeployments = new HashSet<>();
for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) {
- ApplicationId app = f.getKey();
try {
f.getValue().get();
} catch (ExecutionException e) {
+ ApplicationId app = f.getKey();
if (e.getCause() instanceof TransientException) {
log.log(LogLevel.INFO, "Redeploying " + app +
" failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
index ea835206b7c..d46c551fa7e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
@@ -11,6 +11,7 @@ import com.yahoo.config.model.api.SuperModelProvider;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.config.provision.Zone;
+import com.yahoo.log.LogLevel;
import com.yahoo.vespa.config.GenerationCounter;
import com.yahoo.vespa.config.server.application.ApplicationSet;
import com.yahoo.vespa.config.server.model.SuperModelConfigProvider;
@@ -18,14 +19,19 @@ import com.yahoo.vespa.flags.FlagSource;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
/**
* Provides a SuperModel - a model of all application instances, and makes it stays
* up to date as applications are added, redeployed, and removed.
*/
public class SuperModelManager implements SuperModelProvider {
+ private static final Logger logger = Logger.getLogger(SuperModelManager.class.getName());
+
private final Zone zone;
private final Object monitor = new Object();
@@ -38,6 +44,9 @@ public class SuperModelManager implements SuperModelProvider {
private final long masterGeneration; // ConfigserverConfig's generation
private final GenerationCounter generationCounter;
+ // The initial set of applications to be deployed on bootstrap.
+ private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty();
+
@Inject
public SuperModelManager(ConfigserverConfig configserverConfig,
NodeFlavors nodeFlavors,
@@ -75,6 +84,10 @@ public class SuperModelManager implements SuperModelProvider {
listeners.add(listener);
SuperModel superModel = superModelConfigProvider.getSuperModel();
superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application));
+
+ if (superModel.isComplete()) {
+ listener.notifyOfCompleteness(superModel);
+ }
}
}
@@ -89,28 +102,37 @@ public class SuperModelManager implements SuperModelProvider {
.getForVersionOrLatest(Optional.empty(), Instant.now())
.toApplicationInfo();
- SuperModel newSuperModel = this.superModelConfigProvider
- .getSuperModel()
+ SuperModel newSuperModel = superModelConfigProvider.getSuperModel()
.cloneAndSetApplication(applicationInfo);
+
generationCounter.increment();
makeNewSuperModelConfigProvider(newSuperModel);
- listeners.stream().forEach(listener ->
- listener.applicationActivated(newSuperModel, applicationInfo));
+ listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo));
}
}
public void applicationRemoved(ApplicationId applicationId) {
synchronized (monitor) {
+ bootstrapApplicationSet.ifPresent(set -> set.remove(applicationId));
+
SuperModel newSuperModel = this.superModelConfigProvider
.getSuperModel()
.cloneAndRemoveApplication(applicationId);
generationCounter.increment();
makeNewSuperModelConfigProvider(newSuperModel);
- listeners.stream().forEach(listener ->
- listener.applicationRemoved(newSuperModel, applicationId));
+ listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId));
}
}
+ public void markAsComplete() {
+ // Invoked on component graph bootstrap (even before ConfigServerBootstrap),
+ // there is no need to bump generation counter.
+ logger.log(LogLevel.INFO, "Super model is complete");
+ SuperModel newSuperModel = getSuperModel().cloneAsComplete();
+ superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
+ listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel));
+ }
+
private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) {
generation = masterGeneration + generationCounter.get();
superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java
index 520972f2fcf..51c5be1b1f0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java
@@ -125,5 +125,7 @@ public class SuperModelRequestHandler implements RequestHandler {
public void enable() {
enabled = true;
+ superModelManager.markAsComplete();
+ updateHandler();
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 7ec0f49ed60..7ff5d41485d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -21,10 +21,8 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import java.time.Duration;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.logging.Logger;
@@ -49,7 +47,6 @@ public class TenantApplications {
private final Path locksPath;
private final Curator.DirectoryCache directoryCache;
private final ReloadHandler reloadHandler;
- private final Map<ApplicationId, Lock> locks;
private final Executor zkWatcherExecutor;
private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant,
@@ -57,7 +54,6 @@ public class TenantApplications {
this.curator = curator;
this.applicationsPath = TenantRepository.getApplicationsPath(tenant);
this.locksPath = TenantRepository.getLocksPath(tenant);
- this.locks = new ConcurrentHashMap<>(2);
this.reloadHandler = reloadHandler;
this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command);
this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor);
@@ -148,10 +144,7 @@ public class TenantApplications {
/** Returns the lock for changing the session status of the given application. */
public Lock lock(ApplicationId id) {
- curator.create(lockPath(id));
- Lock lock = locks.computeIfAbsent(id, __ -> new Lock(lockPath(id).getAbsolute(), curator));
- lock.acquire(Duration.ofMinutes(1)); // These locks shouldn't be held for very long.
- return lock;
+ return curator.lock(lockPath(id), Duration.ofMinutes(1)); // These locks shouldn't be held for very long.
}
private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index 89d7c349d6b..4bd5cf30cc6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -4,10 +4,9 @@ package com.yahoo.vespa.config.server.deploy;
import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.HostFilter;
import com.yahoo.config.provision.Provisioner;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.jdisc.Metric;
import com.yahoo.log.LogLevel;
import com.yahoo.transaction.NestedTransaction;
import com.yahoo.transaction.Transaction;
@@ -25,7 +24,6 @@ import com.yahoo.vespa.curator.Lock;
import java.time.Clock;
import java.time.Duration;
-import java.time.Instant;
import java.util.Optional;
import java.util.logging.Logger;
@@ -49,13 +47,19 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
private final Duration timeout;
private final Clock clock;
private final DeployLogger logger = new SilentDeployLogger();
-
+
+ /** The repository part of docker image this application should run on. Version is separate from image repo */
+ Optional<String> dockerImageRepository;
+
/** The Vespa version this application should run on */
private final Version version;
/** True if this deployment is done to bootstrap the config server */
private final boolean isBootstrap;
+ /** The (optional) Athenz domain this application should use */
+ private final Optional<AthenzDomain> athenzDomain;
+
private boolean prepared = false;
/** Whether this model should be validated (only takes effect if prepared=false) */
@@ -64,9 +68,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
private boolean ignoreSessionStaleFailure = false;
private Deployment(LocalSession session, ApplicationRepository applicationRepository,
- Optional<Provisioner> hostProvisioner, Tenant tenant,
- Duration timeout, Clock clock, boolean prepared, boolean validate, Version version,
- boolean isBootstrap) {
+ Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout,
+ Clock clock, boolean prepared, boolean validate, boolean isBootstrap) {
this.session = session;
this.applicationRepository = applicationRepository;
this.hostProvisioner = hostProvisioner;
@@ -75,23 +78,24 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
this.clock = clock;
this.prepared = prepared;
this.validate = validate;
- this.version = version;
+ this.dockerImageRepository = session.getDockerImageRepository();
+ this.version = session.getVespaVersion();
this.isBootstrap = isBootstrap;
+ this.athenzDomain = session.getAthenzDomain();
}
public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository,
Optional<Provisioner> hostProvisioner, Tenant tenant,
- Duration timeout, Clock clock, boolean validate, Version version,
- boolean isBootstrap) {
- return new Deployment(session, applicationRepository, hostProvisioner, tenant,
- timeout, clock, false, validate, version, isBootstrap);
+ Duration timeout, Clock clock, boolean validate, boolean isBootstrap) {
+ return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false,
+ validate, isBootstrap);
}
public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository,
Optional<Provisioner> hostProvisioner, Tenant tenant,
Duration timeout, Clock clock, boolean isBootstrap) {
return new Deployment(session, applicationRepository, hostProvisioner, tenant,
- timeout, clock, true, true, session.getVespaVersion(), isBootstrap);
+ timeout, clock, true, true, isBootstrap);
}
public void setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) {
@@ -105,16 +109,14 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.prepareMillis")) {
TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout);
- session.prepare(logger,
- new PrepareParams.Builder().applicationId(session.getApplicationId())
- .timeoutBudget(timeoutBudget)
- .ignoreValidationErrors(!validate)
- .vespaVersion(version.toString())
- .isBootstrap(isBootstrap)
- .build(),
- Optional.empty(),
- tenant.getPath(),
- clock.instant());
+ PrepareParams.Builder params = new PrepareParams.Builder().applicationId(session.getApplicationId())
+ .timeoutBudget(timeoutBudget)
+ .ignoreValidationErrors(!validate)
+ .vespaVersion(version.toString())
+ .isBootstrap(isBootstrap);
+ dockerImageRepository.ifPresent(params::dockerImageRepository);
+ athenzDomain.ifPresent(params::athenzDomain);
+ session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant());
this.prepared = true;
}
}
@@ -129,10 +131,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout);
ApplicationId applicationId = session.getApplicationId();
+ LocalSession previousActiveSession;
try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) {
validateSessionStatus(session);
NestedTransaction transaction = new NestedTransaction();
- transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure));
+ previousActiveSession = applicationRepository.getActiveSession(applicationId);
+ transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure));
hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts()));
transaction.commit();
}
@@ -147,8 +151,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() +
" activated successfully using " +
- (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") +
+ (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") +
". Config generation " + session.getMetaData().getGeneration() +
+ (previousActiveSession != null ? ". Activated session based on previous active session " + previousActiveSession.getSessionId() : "") +
". File references used: " + applicationRepository.getFileReferences(applicationId));
}
}
@@ -166,14 +171,13 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
/** Exposes the session of this for testing only */
public LocalSession session() { return session; }
- private long validateSessionStatus(LocalSession localSession) {
+ private void validateSessionStatus(LocalSession localSession) {
long sessionId = localSession.getSessionId();
if (Session.Status.NEW.equals(localSession.getStatus())) {
throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is not prepared");
} else if (Session.Status.ACTIVATE.equals(localSession.getStatus())) {
throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is already active");
}
- return sessionId;
}
private Transaction deactivateCurrentActivateNew(LocalSession active, LocalSession prepared, boolean ignoreStaleSessionFailure) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 23bdf972a06..400460d2ce6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -12,8 +12,10 @@ import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.model.api.Model;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.api.TlsSecrets;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.flags.FetchVector;
@@ -40,9 +42,12 @@ public class ModelContextImpl implements ModelContext {
private final ConfigDefinitionRepo configDefinitionRepo;
private final FileRegistry fileRegistry;
private final Optional<HostProvisioner> hostProvisioner;
+ private final Provisioned provisioned;
private final ModelContext.Properties properties;
private final Optional<File> appDir;
+ private final Optional<String> wantedDockerImageRepository;
+
/** The version of Vespa we are building a model for */
private final Version modelVespaVersion;
@@ -62,8 +67,10 @@ public class ModelContextImpl implements ModelContext {
ConfigDefinitionRepo configDefinitionRepo,
FileRegistry fileRegistry,
Optional<HostProvisioner> hostProvisioner,
+ Provisioned provisioned,
ModelContext.Properties properties,
Optional<File> appDir,
+ Optional<String> wantedDockerImageRepository,
Version modelVespaVersion,
Version wantedNodeVespaVersion) {
this.applicationPackage = applicationPackage;
@@ -73,8 +80,10 @@ public class ModelContextImpl implements ModelContext {
this.configDefinitionRepo = configDefinitionRepo;
this.fileRegistry = fileRegistry;
this.hostProvisioner = hostProvisioner;
+ this.provisioned = provisioned;
this.properties = properties;
this.appDir = appDir;
+ this.wantedDockerImageRepository = wantedDockerImageRepository;
this.modelVespaVersion = modelVespaVersion;
this.wantedNodeVespaVersion = wantedNodeVespaVersion;
}
@@ -97,6 +106,9 @@ public class ModelContextImpl implements ModelContext {
public Optional<HostProvisioner> hostProvisioner() { return hostProvisioner; }
@Override
+ public Provisioned provisioned() { return provisioned; }
+
+ @Override
public DeployLogger deployLogger() { return deployLogger; }
@Override
@@ -112,6 +124,9 @@ public class ModelContextImpl implements ModelContext {
public Optional<File> appDir() { return appDir; }
@Override
+ public Optional<String> wantedDockerImageRepository() { return wantedDockerImageRepository; }
+
+ @Override
public Version modelVespaVersion() { return modelVespaVersion; }
@Override
@@ -131,8 +146,12 @@ public class ModelContextImpl implements ModelContext {
private final boolean isBootstrap;
private final boolean isFirstTimeDeployment;
private final boolean useAdaptiveDispatch;
+ private final double defaultTopKprobability;
private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
private final double defaultTermwiseLimit;
+ private final double defaultSoftStartSeconds;
+ private final String proxyProtocol;
+ private final Optional<AthenzDomain> athenzDomain;
public Properties(ApplicationId applicationId,
boolean multitenantFromConfig,
@@ -146,7 +165,8 @@ public class ModelContextImpl implements ModelContext {
boolean isBootstrap,
boolean isFirstTimeDeployment,
FlagSource flagSource,
- Optional<EndpointCertificateSecrets> endpointCertificateSecrets) {
+ Optional<EndpointCertificateSecrets> endpointCertificateSecrets,
+ Optional<AthenzDomain> athenzDomain) {
this.applicationId = applicationId;
this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant");
this.configServerSpecs = configServerSpecs;
@@ -163,6 +183,13 @@ public class ModelContextImpl implements ModelContext {
this.endpointCertificateSecrets = endpointCertificateSecrets;
defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource)
.with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+ defaultSoftStartSeconds = Flags.DEFAULT_SOFT_START_SECONDS.bindTo(flagSource)
+ .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+ defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource)
+ .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+ this.proxyProtocol = Flags.PROXY_PROTOCOL.bindTo(flagSource)
+ .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+ this.athenzDomain = athenzDomain;
}
@Override
@@ -214,9 +241,28 @@ public class ModelContextImpl implements ModelContext {
@Override
public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
+ public double defaultSoftStartSeconds() {
+ return 0;
+ }
+
+ @Override
+ public double defaultTopKProbability() {
+ return defaultTopKprobability;
+ }
+
// TODO: Remove
@Override
public boolean useBucketSpaceMetric() { return true; }
+
+ @Override
+ public boolean useNewAthenzFilter() { return true; }
+
+ @Override
+ public String proxyProtocol() { return proxyProtocol; }
+
+ @Override
+ public Optional<AthenzDomain> athenzDomain() { return athenzDomain; }
+
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index 13ef19f5f5d..09f5178cf6b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -87,17 +87,16 @@ public class ZooKeeperClient {
/** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */
private void createZooKeeperNodes() {
- if (!configCurator.exists(rootPath.getAbsolute())) {
+ if ( ! configCurator.exists(rootPath.getAbsolute()))
configCurator.createNode(rootPath.getAbsolute());
- }
- for (String subPath : Arrays.asList(
- ConfigCurator.DEFCONFIGS_ZK_SUBPATH,
- ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH,
- ConfigCurator.USERAPP_ZK_SUBPATH,
- ZKApplicationPackage.fileRegistryNode)) {
- // TODO The replaceFirst below is hackish.
- configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", ""));
+ for (String subPath : Arrays.asList(ConfigCurator.DEFCONFIGS_ZK_SUBPATH,
+ ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH,
+ ConfigCurator.USERAPP_ZK_SUBPATH,
+ ZKApplicationPackage.fileRegistryNode)) {
+ // TODO: The replaceFirst below is hackish.
+ configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(),
+ subPath.replaceFirst("/", ""));
}
}
@@ -108,7 +107,6 @@ public class ZooKeeperClient {
*/
void write(ApplicationPackage app) {
logFine("Feeding application config into ZooKeeper");
- // gives lots and lots of debug output: // BasicConfigurator.configure();
try {
logFine("Feeding user def files into ZooKeeper");
writeUserDefs(app);
@@ -121,43 +119,34 @@ public class ZooKeeperClient {
write(app.getMetaData());
} catch (Exception e) {
throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" +
- "Please ensure that cloudconfig_server is started on the config server node(s), " +
- "and check the vespa log for configserver errors. ", e);
+ "Please ensure that cloudconfig_server is started on the config server node(s), " +
+ "and check the vespa log for configserver errors. ", e);
}
}
private void writeSearchDefinitions(ApplicationPackage app) throws IOException {
Collection<NamedReader> sds = app.getSearchDefinitions();
- if (sds.isEmpty()) {
- return;
- }
+ if (sds.isEmpty()) return;
+
+ // TODO: Change to SCHEMAS_DIR after March 2020
Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR);
configCurator.createNode(zkPath.getAbsolute());
- // Ensures that ranking expressions and other files are also fed.
+ // Ensures that ranking expressions and other files are also written
writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
+ writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false);
for (NamedReader sd : sds) {
- String name = sd.getName();
- Reader reader = sd.getReader();
- String data = com.yahoo.io.IOUtils.readAll(reader);
- reader.close();
- configCurator.putData(zkPath.getAbsolute(), name, data);
+ configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader()));
+ sd.getReader().close();
}
}
/**
* Puts some of the application package files into ZK - see write(app).
*
- * @param app The application package to use as input.
- * @throws java.io.IOException if not able to write to Zookeeper
+ * @param app the application package to use as input.
+ * @throws java.io.IOException if not able to write to Zookeeper
*/
private void writeSomeOf(ApplicationPackage app) throws IOException {
- ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX);
- }
- };
- // Copy app package files and subdirs into zk
// TODO: We should have a way of doing this which doesn't require repeating all the content
writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)),
getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH));
@@ -169,7 +158,8 @@ public class ZooKeeperClient {
getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH));
writeDir(app.getFile(ApplicationPackage.RULES_DIR),
getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR),
- srFilter, true);
+ (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX),
+ true);
writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR),
getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR),
xmlFilter, true);
@@ -194,20 +184,12 @@ public class ZooKeeperClient {
}
private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException {
- writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() {
- @Override
- public boolean accept(Path path) {
- return true;
- }
- }, recurse);
+ writeDir(file, zooKeeperAppPath, (__) -> true, recurse);
}
private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException {
- if (!dir.isDirectory()) {
- logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper.");
- return;
- }
- for (ApplicationFile file: listFiles(dir, filenameFilter)) {
+ if ( ! dir.isDirectory()) return;
+ for (ApplicationFile file : listFiles(dir, filenameFilter)) {
String name = file.getPath().getName();
if (name.startsWith(".")) continue; //.svn , .git ...
if ("CVS".equals(name)) continue;
@@ -223,7 +205,8 @@ public class ZooKeeperClient {
}
/**
- * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories.
+ * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)}
+ * with slightly different semantics: Never filter out directories.
*/
private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) {
List<ApplicationFile> rawList = dir.listFiles();
@@ -243,9 +226,8 @@ public class ZooKeeperClient {
}
private void writeFile(ApplicationFile file, Path zkPath) throws IOException {
- if (!file.exists()) {
- return;
- }
+ if ( ! file.exists()) return;
+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (InputStream inputStream = file.createInputStream()) {
inputStream.transferTo(baos);
@@ -292,8 +274,8 @@ public class ZooKeeperClient {
String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry);
configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(),
- vespaVersion.toFullString(),
- exportedRegistry);
+ vespaVersion.toFullString(),
+ exportedRegistry);
}
/**
@@ -343,9 +325,8 @@ public class ZooKeeperClient {
}
public void write(AllocatedHosts hosts) throws IOException {
- configCurator.putData(
- rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
- AllocatedHostsSerializer.toJson(hosts));
+ configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
+ AllocatedHostsSerializer.toJson(hosts));
}
public void write(Map<Version, FileRegistry> fileRegistryMap) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
index 1418b2a0bcf..2d5cce2d4f1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
@@ -59,8 +59,7 @@ public class TesterClient {
}
private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) {
- // TODO: Change log level to DEBUG
- logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString());
+ logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString());
try {
return new ProxyResponse(httpClient.execute(request));
} catch (IOException e) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java
index bb752f9b804..0014d66026b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java
@@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.slime.Cursor;
import com.yahoo.vespa.config.ConfigPayload;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.config.server.GlobalComponentRegistry;
import com.yahoo.vespa.config.server.http.HttpHandler;
import com.yahoo.vespa.config.server.http.JSONResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 833c9ca4fba..3c909730902 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -245,7 +245,6 @@ public class ApplicationHandler extends HttpHandler {
}
private static boolean isTesterStartTestsRequest(HttpRequest request) {
- System.out.println(getBindingMatch(request).groupCount());
return getBindingMatch(request).groupCount() == 9 &&
request.getUri().getPath().contains("/tester/run/");
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
index d2fd4efa7dc..e40d9153ad1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
@@ -48,7 +48,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
@Override
@SuppressWarnings({"try", "unused"})
public void run() {
- try (Lock lock = lock(lockRoot.append(name()))) {
+ try (Lock lock = curator.lock(lockRoot.append(name()), Duration.ofSeconds(1))) {
maintain();
} catch (UncheckedTimeoutException e) {
// another config server instance is running this job at the moment; ok
@@ -57,12 +57,6 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
}
}
- private Lock lock(Path path) {
- Lock lock = new Lock(path.getAbsolute(), curator);
- lock.acquire(Duration.ofSeconds(1));
- return lock;
- }
-
@Override
public void deconstruct() {
this.service.shutdown();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java
index 34c7dced404..4f64b73b403 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java
@@ -6,12 +6,13 @@ import com.yahoo.log.LogLevel;
import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.yolean.Exceptions;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import java.io.IOException;
import java.io.InputStream;
@@ -42,12 +43,15 @@ public class ClusterMetricsRetriever {
private static final List<String> WANTED_METRIC_SERVICES = List.of(VESPA_CONTAINER, VESPA_QRSERVER, VESPA_DISTRIBUTOR);
- private static final CloseableHttpClient httpClient = VespaHttpClientBuilder.create()
- .setDefaultRequestConfig(RequestConfig.custom()
- .setConnectTimeout(10 * 1000)
- .setSocketTimeout(10 * 1000)
- .build())
- .build();
+
+ private static final CloseableHttpClient httpClient = VespaHttpClientBuilder
+ .create(PoolingHttpClientConnectionManager::new)
+ .setDefaultRequestConfig(
+ RequestConfig.custom()
+ .setConnectTimeout(10 * 1000)
+ .setSocketTimeout(10 * 1000)
+ .build())
+ .build();
/**
* Call the metrics API on each host and aggregate the metrics
@@ -62,7 +66,7 @@ public class ClusterMetricsRetriever {
getHostMetrics(host, clusterMetricsMap)
);
- ForkJoinPool threadPool = new ForkJoinPool(5);
+ ForkJoinPool threadPool = new ForkJoinPool(10);
threadPool.submit(retrieveMetricsJob);
threadPool.shutdown();
@@ -102,7 +106,7 @@ public class ClusterMetricsRetriever {
return slime;
} catch (IOException e) {
// Usually caused by applications being deleted during metric retrieval
- log.warning("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e));
+ log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e));
return new Slime();
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java
index d88fae0a8ef..4fe2ec129d3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java
@@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.flags.BooleanFlag;
-import com.yahoo.vespa.flags.FetchVector;
import com.yahoo.vespa.flags.FlagSource;
import com.yahoo.vespa.flags.Flags;
@@ -35,16 +34,17 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
private final Map<TenantName, Set<ApplicationInfo>> models;
private final Zone zone;
- private final BooleanFlag use4443Upstream;
+ private final BooleanFlag nginxUpstreamProxyProtocol;
public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) {
this.models = models;
this.zone = zone;
- this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource);
+ this.nginxUpstreamProxyProtocol = Flags.NGINX_UPSTREAM_PROXY_PROTOCOL.bindTo(flagSource);
}
@Override
public void getConfig(LbServicesConfig.Builder builder) {
+ builder.nginxUpstreamProxyProtocol(nginxUpstreamProxyProtocol.value());
models.keySet().stream()
.sorted()
.forEach(tenant -> {
@@ -56,10 +56,15 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
LbServicesConfig.Tenants.Builder tb = new LbServicesConfig.Tenants.Builder();
apps.stream()
.sorted(Comparator.comparing(ApplicationInfo::getApplicationId))
+ .filter(applicationInfo -> generateRoutingConfig(applicationInfo.getApplicationId()))
.forEach(applicationInfo -> tb.applications(createLbAppIdKey(applicationInfo.getApplicationId()), getAppConfig(applicationInfo)));
return tb;
}
+ private boolean generateRoutingConfig(ApplicationId applicationId) {
+ return ( ! applicationId.instance().isTester());
+ }
+
private String createLbAppIdKey(ApplicationId applicationId) {
return applicationId.application().value() + ":" + zone.environment().value() + ":" + zone.region().value() + ":" + applicationId.instance().value();
}
@@ -67,8 +72,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) {
LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder();
ab.activeRotation(getActiveRotation(app));
- ab.use4443Upstream(
- use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value());
app.getModel().getHosts().stream()
.sorted((a, b) -> a.getHostname().compareTo(b.getHostname()))
.forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo)));
@@ -92,10 +95,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) {
LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder();
hb.hostname(hostInfo.getHostname());
- hostInfo.getServices().stream()
- .forEach(serviceInfo -> {
- hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo));
- });
+ hostInfo.getServices().forEach(serviceInfo -> hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo)));
return hb;
}
@@ -114,12 +114,11 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
filter(prop -> !"".equals(prop)).sorted((a, b) -> a.compareTo(b)).collect(Collectors.toList()))
.endpointaliases(endpointAliases)
.index(Integer.parseInt(serviceInfo.getProperty("index").orElse("999999")));
- serviceInfo.getPorts().stream()
- .forEach(portInfo -> {
- LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder()
- .number(portInfo.getPort())
- .tags(Joiner.on(" ").join(portInfo.getTags()));
- sb.ports(pb);
+ serviceInfo.getPorts().forEach(portInfo -> {
+ LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder()
+ .number(portInfo.getPort())
+ .tags(Joiner.on(" ").join(portInfo.getTags()));
+ sb.ports(pb);
});
return sb;
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
index a2fc2bfd6a0..99bec2c9db7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
@@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.api.ConfigDefinitionRepo;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.provision.AllocatedHosts;
import com.yahoo.config.provision.ApplicationId;
@@ -83,12 +84,14 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
protected Application buildModelVersion(ModelFactory modelFactory,
ApplicationPackage applicationPackage,
ApplicationId applicationId,
+ Optional<String> wantedDockerImageRepository,
Version wantedNodeVespaVersion,
Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models
Instant now) {
log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s",
modelFactory.version(), appGeneration, applicationId));
ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId);
+ Provisioned provisioned = new Provisioned();
ModelContext modelContext = new ModelContextImpl(
applicationPackage,
Optional.empty(),
@@ -96,9 +99,13 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
logger,
configDefinitionRepo,
getForVersionOrLatest(applicationPackage.getFileRegistries(), modelFactory.version()).orElse(new MockFileRegistry()),
- createStaticProvisioner(applicationPackage.getAllocatedHosts(), modelContextProperties),
+ createStaticProvisioner(applicationPackage.getAllocatedHosts(),
+ modelContextProperties.applicationId(),
+ provisioned),
+ provisioned,
modelContextProperties,
Optional.empty(),
+ wantedDockerImageRepository,
modelFactory.version(),
wantedNodeVespaVersion);
MetricUpdater applicationMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(applicationId));
@@ -138,7 +145,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
flagSource,
new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant))
.readEndpointCertificateMetadata(applicationId)
- .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets));
+ .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets),
+ zkClient.readAthenzDomain());
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 2de64ed145c..cc4378ae05e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.api.HostProvisioner;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.provision.AllocatedHosts;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ApplicationLockException;
@@ -72,6 +73,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
* and assigns to this SettableOptional such that it can be used after this method returns
*/
public List<MODELRESULT> buildModels(ApplicationId applicationId,
+ Optional<String> dockerImageRepository,
Version wantedNodeVespaVersion,
ApplicationPackage applicationPackage,
SettableOptional<AllocatedHosts> allocatedHosts,
@@ -104,8 +106,9 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
int majorVersion = majorVersions.get(i);
try {
allApplicationModels.addAll(buildModelVersions(keepMajorVersion(majorVersion, versions),
- applicationId, wantedNodeVespaVersion, applicationPackage,
- allocatedHosts, now, buildLatestModelForThisMajor, majorVersion));
+ applicationId, dockerImageRepository, wantedNodeVespaVersion,
+ applicationPackage, allocatedHosts, now,
+ buildLatestModelForThisMajor, majorVersion));
buildLatestModelForThisMajor = false; // We have successfully built latest model version, do it only for this major
}
catch (OutOfCapacityException | ApplicationLockException | TransientException e) {
@@ -146,6 +149,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
// versions is the set of versions for one particular major version
private List<MODELRESULT> buildModelVersions(Set<Version> versions,
ApplicationId applicationId,
+ Optional<String> wantedDockerImageRepository,
Version wantedNodeVespaVersion,
ApplicationPackage applicationPackage,
SettableOptional<AllocatedHosts> allocatedHosts,
@@ -160,6 +164,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
MODELRESULT latestModelVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest.get()),
applicationPackage,
applicationId,
+ wantedDockerImageRepository,
wantedNodeVespaVersion,
allocatedHosts.asOptional(),
now);
@@ -182,6 +187,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
modelVersion = buildModelVersion(modelFactoryRegistry.getFactory(version),
applicationPackage,
applicationId,
+ wantedDockerImageRepository,
wantedNodeVespaVersion,
allocatedHosts.asOptional(),
now);
@@ -192,8 +198,10 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
// config models (which is always true for manually deployed zones)
if (allApplicationVersions.size() > 0 && allApplicationVersions.get(0).getModel().skipOldConfigModels(now))
log.log(LogLevel.INFO, applicationId + ": Skipping old version (due to validation override)");
- else
+ else {
+ log.log(LogLevel.ERROR, applicationId + ": Failed to build version " + version);
throw e;
+ }
}
}
return allApplicationVersions;
@@ -236,9 +244,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
}
protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage,
- ApplicationId applicationId,
- Version wantedNodeVespaVersion,
- Optional<AllocatedHosts> allocatedHosts,
+ ApplicationId applicationId, Optional<String> dockerImageRepository,
+ Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts,
Instant now);
/**
@@ -246,15 +253,17 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
* returns empty otherwise, which may either mean that no hosts are allocated or that we are running
* non-hosted and should default to use hosts defined in the application package, depending on context
*/
- Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts, ModelContext.Properties properties) {
+ Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts,
+ ApplicationId applicationId,
+ Provisioned provisioned) {
if (hosted && allocatedHosts.isPresent())
- return Optional.of(new StaticProvisioner(allocatedHosts.get(), createNodeRepositoryProvisioner(properties).get()));
+ return Optional.of(new StaticProvisioner(allocatedHosts.get(), createNodeRepositoryProvisioner(applicationId, provisioned).get()));
return Optional.empty();
}
- Optional<HostProvisioner> createNodeRepositoryProvisioner(ModelContext.Properties properties) {
+ Optional<HostProvisioner> createNodeRepositoryProvisioner(ApplicationId applicationId, Provisioned provisioned) {
return hostProvisionerProvider.getHostProvisioner().map(
- provisioner -> new ProvisionerAdapter(provisioner, properties.applicationId()));
+ provisioner -> new ProvisionerAdapter(provisioner, applicationId, provisioned));
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
index 9cacb78e53c..6d0403f21c8 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
@@ -11,6 +11,7 @@ import com.yahoo.config.model.api.Model;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.ModelCreateResult;
import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.api.ValidationParameters;
import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors;
import com.yahoo.config.model.application.provider.FilesApplicationPackage;
@@ -82,8 +83,9 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
@Override
protected PreparedModelResult buildModelVersion(ModelFactory modelFactory,
ApplicationPackage applicationPackage,
- ApplicationId applicationId,
- com.yahoo.component.Version wantedNodeVespaVersion,
+ ApplicationId applicationId,
+ Optional<String> wantedDockerImageRepository,
+ Version wantedNodeVespaVersion,
Optional<AllocatedHosts> allocatedHosts,
Instant now) {
Version modelVersion = modelFactory.version();
@@ -91,6 +93,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir());
// Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise
+ Provisioned provisioned = new Provisioned();
ModelContext modelContext = new ModelContextImpl(
applicationPackage,
modelOf(modelVersion),
@@ -98,9 +101,11 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
logger,
configDefinitionRepo,
fileDistributionProvider.getFileRegistry(),
- createHostProvisioner(allocatedHosts),
+ createHostProvisioner(allocatedHosts, provisioned),
+ provisioned,
properties,
getAppDir(applicationPackage),
+ wantedDockerImageRepository,
modelVersion,
wantedNodeVespaVersion);
@@ -120,11 +125,15 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
// This method is an excellent demonstration of what happens when one is too liberal with Optional
// -bratseth, who had to write the below :-\
- private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts) {
- Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties);
+ private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts,
+ Provisioned provisioned) {
+ Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties.applicationId(),
+ provisioned);
if ( ! allocatedHosts.isPresent()) return nodeRepositoryProvisioner;
- Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts, properties);
+ Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts,
+ properties.applicationId(),
+ provisioned);
if ( ! staticProvisioner.isPresent()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted
// Nodes are already allocated by a model and we should use them unless this model requests hosts from a
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java
index 87e9fa287a4..249c8639ea9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.monitoring;
import com.google.inject.Inject;
import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.concurrent.DaemonThreadFactory;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
@@ -14,14 +16,17 @@ import com.yahoo.statistics.Counter;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
/**
- * Statistics for server. The statistics framework takes care of logging.
+ * Metrics for config server. The statistics framework takes care of logging.
*
* @author Harald Musum
- * @since 4.2
*/
-public class Metrics extends TimerTask implements MetricUpdaterFactory {
+public class Metrics extends AbstractComponent implements MetricUpdaterFactory, Runnable {
private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Metrics.class.getName());
private static final String METRIC_REQUESTS = getMetricName("requests");
@@ -33,23 +38,34 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory {
private final Counter failedRequests;
private final Counter procTimeCounter;
private final Metric metric;
- private final ZKMetricUpdater zkMetricUpdater;
+ private final Optional<ZKMetricUpdater> zkMetricUpdater;
// TODO The map is the key for now
private final Map<Map<String, String>, MetricUpdater> metricUpdaters = new ConcurrentHashMap<>();
- private final Timer timer = new Timer();
+ private final Optional<ScheduledExecutorService> executorService;
@Inject
public Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, ZookeeperServerConfig zkServerConfig) {
+ this(metric, statistics, healthMonitorConfig, zkServerConfig, true);
+ }
+
+ private Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig,
+ ZookeeperServerConfig zkServerConfig, boolean createZkMetricUpdater) {
this.metric = metric;
requests = createCounter(METRIC_REQUESTS, statistics);
failedRequests = createCounter(METRIC_FAILED_REQUESTS, statistics);
procTimeCounter = createCounter("procTime", statistics);
- log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds");
- long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000);
- timer.scheduleAtFixedRate(this, 20000, intervalMs);
- zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 19500, intervalMs);
+ if (createZkMetricUpdater) {
+ log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds");
+ long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000);
+ executorService = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("configserver-metrics")));
+ executorService.get().scheduleAtFixedRate(this, 20000, intervalMs, TimeUnit.MILLISECONDS);
+ zkMetricUpdater = Optional.of(new ZKMetricUpdater(zkServerConfig, 19500, intervalMs));
+ } else {
+ executorService = Optional.empty();
+ zkMetricUpdater = Optional.empty();
+ }
}
public static Metrics createTestMetrics() {
@@ -58,14 +74,13 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory {
HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder();
builder.snapshot_interval(60.0);
ZookeeperServerConfig.Builder zkBuilder = new ZookeeperServerConfig.Builder().myid(1);
- return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder));
+ return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder), false);
}
private Counter createCounter(String name, Statistics statistics) {
return new Counter(name, statistics, false);
}
-
void incrementRequests(Metric.Context metricContext) {
requests.increment(1);
metric.add(METRIC_REQUESTS, 1, metricContext);
@@ -126,8 +141,12 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory {
}
}
setRegularMetrics();
- zkMetricUpdater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null));
- timer.purge();
+ zkMetricUpdater.ifPresent(updater -> updater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null)));
+ }
+
+ public void deconstruct() {
+ executorService.ifPresent(ExecutorService::shutdown);
+ zkMetricUpdater.ifPresent(ZKMetricUpdater::shutdown);
}
private void setRegularMetrics() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java
index b2813be5456..62da6fcffbe 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.config.server.monitoring;
import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.concurrent.DaemonThreadFactory;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -17,6 +18,8 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
@@ -26,7 +29,7 @@ import java.util.regex.Pattern;
import static com.yahoo.vespa.config.server.monitoring.Metrics.getMetricName;
-public class ZKMetricUpdater extends TimerTask {
+public class ZKMetricUpdater implements Runnable {
private static final Logger log = Logger.getLogger(ZKMetricUpdater.class.getName());
public static final String METRIC_ZK_ZNODES = getMetricName("zkZNodes");
@@ -35,19 +38,19 @@ public class ZKMetricUpdater extends TimerTask {
public static final String METRIC_ZK_CONNECTIONS = getMetricName("zkConnections");
public static final String METRIC_ZK_OUTSTANDING_REQUESTS = getMetricName("zkOutstandingRequests");
- private final int CONNECTION_TIMEOUT_MS = 500;
- private final int WRITE_TIMEOUT_MS = 250;
- private final int READ_TIMEOUT_MS = 500;
+ private static final int CONNECTION_TIMEOUT_MS = 500;
+ private static final int WRITE_TIMEOUT_MS = 250;
+ private static final int READ_TIMEOUT_MS = 500;
private AtomicReference<Map<String, Long>> zkMetrics = new AtomicReference<>(new HashMap<>());
- private final Timer timer = new Timer();
+ private final ScheduledExecutorService executorService;
private final int zkPort;
public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delayMS, long intervalMS) {
this.zkPort = zkServerConfig.clientPort();
- if (intervalMS > 0) {
- timer.scheduleAtFixedRate(this, delayMS, intervalMS);
- }
+ if (intervalMS <= 0 ) throw new IllegalArgumentException("interval must be positive, was " + intervalMS + " ms");
+ this.executorService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("zkmetricupdater"));
+ this.executorService.scheduleAtFixedRate(this, delayMS, intervalMS, TimeUnit.MILLISECONDS);
}
private void setMetricAttribute(String attribute, long value, Map<String, Long> data) {
@@ -74,7 +77,10 @@ public class ZKMetricUpdater extends TimerTask {
public void run() {
Optional<String> report = retrieveReport();
report.ifPresent(this::parseReport);
- timer.purge();
+ }
+
+ public void shutdown() {
+ executorService.shutdown();
}
private Optional<String> retrieveReport() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java
index ee4cc4a3043..7ce255910e6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.config.server.provision;
import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterSpec;
@@ -23,10 +24,12 @@ public class ProvisionerAdapter implements HostProvisioner {
private final Provisioner provisioner;
private final ApplicationId applicationId;
+ private final Provisioned provisioned;
- public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId) {
+ public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId, Provisioned provisioned) {
this.provisioner = provisioner;
this.applicationId = applicationId;
+ this.provisioned = provisioned;
}
@Override
@@ -36,8 +39,18 @@ public class ProvisionerAdapter implements HostProvisioner {
}
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
- return provisioner.prepare(applicationId, cluster, capacity, groups, logger);
+ provisioned.add(cluster.id(), capacity);
+ return provisioner.prepare(applicationId, cluster, capacity.withGroups(groups), logger);
}
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+ provisioned.add(cluster.id(), capacity);
+ return provisioner.prepare(applicationId, cluster, capacity, logger);
+ }
+
+
+
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java
index 26d322665f0..8ddfb1e8b09 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java
@@ -33,8 +33,15 @@ public class StaticProvisioner implements HostProvisioner {
throw new UnsupportedOperationException("Allocating a single host from provisioning info is not supported");
}
+
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+ return prepare(cluster, capacity.withGroups(groups), logger);
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
List<HostSpec> hostsAlreadyAllocatedToCluster =
allocatedHosts.getHosts().stream()
.filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster))
@@ -42,7 +49,7 @@ public class StaticProvisioner implements HostProvisioner {
if ( ! hostsAlreadyAllocatedToCluster.isEmpty())
return hostsAlreadyAllocatedToCluster;
else
- return fallback.prepare(cluster, capacity, groups, logger);
+ return fallback.prepare(cluster, capacity, logger);
}
private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
index 67aa49ea0b9..48580dbc6f4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
@@ -167,7 +167,7 @@ class GetConfigProcessor implements Runnable {
log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName());
ConfigPayload emptyPayload = ConfigPayload.empty();
String configMd5 = ConfigUtils.getMd5(emptyPayload);
- ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5);
+ ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5);
request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5());
respond(request);
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
index dcbc21e536c..5235a2bcadd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
@@ -29,7 +29,7 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory {
String configMd5 = ConfigUtils.getMd5(rawPayload);
CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength());
Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes()));
- return new SlimeConfigResponse(compressed, defFile, generation, internalRedeploy, configMd5, info);
+ return new SlimeConfigResponse(compressed, generation, internalRedeploy, configMd5, info);
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
index 91809f90a70..bd0b117c3db 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
@@ -25,7 +25,7 @@ public class UncompressedConfigResponseFactory implements ConfigResponseFactory
Utf8Array rawPayload = payload.toUtf8Array(true);
String configMd5 = ConfigUtils.getMd5(rawPayload);
CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength());
- return new SlimeConfigResponse(rawPayload, defFile, generation, internalRedeploy, configMd5, info);
+ return new SlimeConfigResponse(rawPayload, generation, internalRedeploy, configMd5, info);
}
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
index 2c6d7de8b0c..825ae0d8d92 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.ApplicationMetaData;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.transaction.AbstractTransaction;
import com.yahoo.transaction.NestedTransaction;
import com.yahoo.transaction.Transaction;
@@ -138,6 +139,14 @@ public class LocalSession extends Session implements Comparable<LocalSession> {
zooKeeperClient.writeVespaVersion(version);
}
+ public void setDockerImageRepository(Optional<String> dockerImageRepository) {
+ zooKeeperClient.writeDockerImageRepository(dockerImageRepository);
+ }
+
+ public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) {
+ zooKeeperClient.writeAthenzDomain(athenzDomain);
+ }
+
public enum Mode {
READ, WRITE
}
@@ -148,8 +157,12 @@ public class LocalSession extends Session implements Comparable<LocalSession> {
public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); }
+ public Optional<String> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); }
+
public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); }
+ public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); }
+
public AllocatedHosts getAllocatedHosts() {
return zooKeeperClient.getAllocatedHosts();
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index 1a41c1efd7a..f7ed801ddbd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -5,10 +5,11 @@ import com.yahoo.component.Version;
import com.yahoo.config.model.api.ContainerEndpoint;
import com.yahoo.config.model.api.EndpointCertificateMetadata;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.config.server.TimeoutBudget;
import com.yahoo.vespa.config.server.http.SessionHandler;
import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer;
@@ -35,6 +36,8 @@ public final class PrepareParams {
static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints";
static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName";
static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata";
+ static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository";
+ static final String ATHENZ_DOMAIN = "athenzDomain";
private final ApplicationId applicationId;
private final TimeoutBudget timeoutBudget;
@@ -46,11 +49,14 @@ public final class PrepareParams {
private final List<ContainerEndpoint> containerEndpoints;
private final Optional<String> tlsSecretsKeyName;
private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
+ private final Optional<String> dockerImageRepository;
+ private final Optional<AthenzDomain> athenzDomain;
private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors,
boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion,
List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName,
- Optional<EndpointCertificateMetadata> endpointCertificateMetadata) {
+ Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
+ Optional<String> dockerImageRepository, Optional<AthenzDomain> athenzDomain) {
this.timeoutBudget = timeoutBudget;
this.applicationId = applicationId;
this.ignoreValidationErrors = ignoreValidationErrors;
@@ -61,6 +67,8 @@ public final class PrepareParams {
this.containerEndpoints = containerEndpoints;
this.tlsSecretsKeyName = tlsSecretsKeyName;
this.endpointCertificateMetadata = endpointCertificateMetadata;
+ this.dockerImageRepository = dockerImageRepository;
+ this.athenzDomain = athenzDomain;
}
public static class Builder {
@@ -75,6 +83,8 @@ public final class PrepareParams {
private List<ContainerEndpoint> containerEndpoints = List.of();
private Optional<String> tlsSecretsKeyName = Optional.empty();
private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty();
+ private Optional<String> dockerImageRepository = Optional.empty();
+ private Optional<AthenzDomain> athenzDomain = Optional.empty();
public Builder() { }
@@ -142,9 +152,26 @@ public final class PrepareParams {
return this;
}
+ public Builder dockerImageRepository(String dockerImageRepository) {
+ if (dockerImageRepository == null) return this;
+ this.dockerImageRepository = Optional.of(dockerImageRepository);
+ return this;
+ }
+
+ public Builder athenzDomain(String athenzDomain) {
+ this.athenzDomain = Optional.ofNullable(athenzDomain).map(AthenzDomain::from);
+ return this;
+ }
+
+ public Builder athenzDomain(AthenzDomain athenzDomain) {
+ this.athenzDomain = Optional.of(athenzDomain);
+ return this;
+ }
+
public PrepareParams build() {
return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun,
- verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata);
+ verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName,
+ endpointCertificateMetadata, dockerImageRepository, athenzDomain);
}
}
@@ -159,6 +186,8 @@ public final class PrepareParams {
.containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME))
.tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME))
.endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME))
+ .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY))
+ .athenzDomain(request.getProperty(ATHENZ_DOMAIN))
.build();
}
@@ -219,4 +248,11 @@ public final class PrepareParams {
public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() {
return endpointCertificateMetadata;
}
+
+ public Optional<String> dockerImageRepository() {
+ return dockerImageRepository;
+ }
+
+ public Optional<AthenzDomain> athenzDomain() { return athenzDomain; }
+
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
index 26f437920ad..39c15474f0e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
@@ -63,6 +63,7 @@ public class RemoteSession extends Session {
Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts();
return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(),
+ zooKeeperClient.readDockerImageRepository(),
zooKeeperClient.readVespaVersion(),
applicationPackage,
new SettableOptional<>(allocatedHosts),
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java
index cb1f6519f57..5ae1289033d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java
@@ -127,10 +127,13 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader {
ApplicationId existingApplicationId = existingSession.getApplicationId();
long activeSessionId = getActiveSessionId(existingApplicationId);
- logger.log(LogLevel.DEBUG, "Create from existing application id " + existingApplicationId + ", active session id is " + activeSessionId);
+ logger.log(LogLevel.DEBUG, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId);
LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget);
+ // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper()
session.setApplicationId(existingApplicationId);
session.setVespaVersion(existingSession.getVespaVersion());
+ session.setDockerImageRepository(existingSession.getDockerImageRepository());
+ session.setAthenzDomain(existingSession.getAthenzDomain());
return session;
}
@@ -138,7 +141,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader {
boolean internalRedeploy, TimeoutBudget timeoutBudget) {
long sessionId = sessionCounter.nextSessionId();
Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId));
- log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Next session id is " + sessionId + " , sessionIdPath=" + sessionIdPath.getAbsolute());
try {
ensureZKPathDoesNotExist(sessionIdPath);
SessionZooKeeperClient sessionZooKeeperClient = new SessionZooKeeperClient(curator,
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 0115876ded9..6c77964a58b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -17,6 +17,7 @@ import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.api.EndpointCertificateSecrets;
import com.yahoo.config.provision.AllocatedHosts;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.Zone;
import com.yahoo.container.jdisc.secretstore.SecretStore;
@@ -139,8 +140,11 @@ public class SessionPreparer {
final Path tenantPath;
final ApplicationId applicationId;
+ /** The repository part of docker image to be used for this deployment */
+ final Optional<String> dockerImageRepository;
+
/** The version of Vespa the application to be prepared specifies for its nodes */
- final com.yahoo.component.Version vespaVersion;
+ final Version vespaVersion;
final ContainerEndpointsCache containerEndpoints;
final Set<ContainerEndpoint> endpointsSet;
@@ -149,6 +153,7 @@ public class SessionPreparer {
private final EndpointCertificateRetriever endpointCertificateRetriever;
private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
+ private final Optional<AthenzDomain> athenzDomain;
private ApplicationPackage applicationPackage;
private List<PreparedModelsBuilder.PreparedModelResult> modelResultList;
@@ -165,6 +170,7 @@ public class SessionPreparer {
this.tenantPath = tenantPath;
this.applicationId = params.getApplicationId();
+ this.dockerImageRepository = params.dockerImageRepository();
this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion);
this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator);
this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath);
@@ -178,6 +184,7 @@ public class SessionPreparer {
.flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
this.endpointsSet = getEndpoints(params.containerEndpoints());
+ this.athenzDomain = params.athenzDomain();
this.properties = new ModelContextImpl.Properties(params.getApplicationId(),
configserverConfig.multitenant(),
@@ -191,7 +198,8 @@ public class SessionPreparer {
params.isBootstrap(),
! currentActiveApplicationSet.isPresent(),
context.getFlagSource(),
- endpointCertificateSecrets);
+ endpointCertificateSecrets,
+ athenzDomain);
this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry,
permanentApplicationPackage,
configDefinitionRepo,
@@ -223,7 +231,7 @@ public class SessionPreparer {
AllocatedHosts buildModels(Instant now) {
SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>();
- this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion,
+ this.modelResultList = preparedModelsBuilder.buildModels(applicationId, dockerImageRepository, vespaVersion,
applicationPackage, allocatedHosts, now);
checkTimeout("build models");
return allocatedHosts.get();
@@ -239,10 +247,12 @@ public class SessionPreparer {
writeStateToZooKeeper(context.getSessionZooKeeperClient(),
applicationPackage,
applicationId,
+ dockerImageRepository,
vespaVersion,
logger,
prepareResult.getFileRegistries(),
- prepareResult.allocatedHosts());
+ prepareResult.allocatedHosts(),
+ athenzDomain);
checkTimeout("write state to zookeeper");
}
@@ -281,15 +291,20 @@ public class SessionPreparer {
private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient,
ApplicationPackage applicationPackage,
ApplicationId applicationId,
- com.yahoo.component.Version vespaVersion,
+ Optional<String> dockerImageRepository,
+ Version vespaVersion,
DeployLogger deployLogger,
Map<Version, FileRegistry> fileRegistryMap,
- AllocatedHosts allocatedHosts) {
+ AllocatedHosts allocatedHosts,
+ Optional<AthenzDomain> athenzDomain) {
ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger);
try {
zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts);
+ // Note: When changing the below you need to also change similar calls in SessionFactoryImpl.createSessionFromExisting()
zooKeeperClient.writeApplicationId(applicationId);
zooKeeperClient.writeVespaVersion(vespaVersion);
+ zooKeeperClient.writeDockerImageRepository(dockerImageRepository);
+ zooKeeperClient.writeAthenzDomain(athenzDomain);
} catch (RuntimeException | IOException e) {
zkDeployer.cleanup();
throw new RuntimeException("Error preparing session", e);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index 727d4e6d7bc..81777bf3642 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -5,23 +5,24 @@ import com.yahoo.component.Version;
import com.yahoo.component.Vtag;
import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.model.api.ConfigDefinitionRepo;
import com.yahoo.config.provision.AllocatedHosts;
-import com.yahoo.transaction.NestedTransaction;
-import com.yahoo.transaction.Transaction;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.NodeFlavors;
import com.yahoo.log.LogLevel;
import com.yahoo.path.Path;
-import com.yahoo.config.model.api.ConfigDefinitionRepo;
import com.yahoo.text.Utf8;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.transaction.Transaction;
import com.yahoo.vespa.config.server.UserConfigDefinitionRepo;
import com.yahoo.vespa.config.server.deploy.ZooKeeperClient;
import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer;
+import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.transaction.CuratorOperations;
import com.yahoo.vespa.curator.transaction.CuratorTransaction;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -41,6 +42,8 @@ public class SessionZooKeeperClient {
static final String APPLICATION_ID_PATH = "applicationId";
private static final String VERSION_PATH = "version";
private static final String CREATE_TIME_PATH = "createTime";
+ private static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository";
+ private static final String ATHENZ_DOMAIN = "athenzDomain";
private final Curator curator;
private final ConfigCurator configCurator;
private final Path sessionPath;
@@ -165,6 +168,14 @@ public class SessionZooKeeperClient {
return sessionPath.append(VERSION_PATH).getAbsolute();
}
+ private String dockerImageRepositoryPath() {
+ return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH).getAbsolute();
+ }
+
+ private String athenzDomainPath() {
+ return sessionPath.append(ATHENZ_DOMAIN).getAbsolute();
+ }
+
public void writeVespaVersion(Version version) {
configCurator.putData(versionPath(), version.toString());
}
@@ -174,6 +185,16 @@ public class SessionZooKeeperClient {
return new Version(configCurator.getData(versionPath()));
}
+ public Optional<String> readDockerImageRepository() {
+ if ( ! configCurator.exists(dockerImageRepositoryPath())) return Optional.empty();
+ String dockerImageRepository = configCurator.getData(dockerImageRepositoryPath());
+ return dockerImageRepository.isEmpty() ? Optional.empty() : Optional.of(dockerImageRepository);
+ }
+
+ public void writeDockerImageRepository(Optional<String> dockerImageRepository) {
+ dockerImageRepository.ifPresent(repo -> configCurator.putData(dockerImageRepositoryPath(), repo));
+ }
+
// in seconds
public long readCreateTime() {
String path = getCreateTimePath();
@@ -206,6 +227,17 @@ public class SessionZooKeeperClient {
return transaction;
}
+ public void writeAthenzDomain(Optional<AthenzDomain> athenzDomain) {
+ athenzDomain.ifPresent(domain -> configCurator.putData(athenzDomainPath(), domain.value()));
+ }
+
+ public Optional<AthenzDomain> readAthenzDomain() {
+ if ( ! configCurator.exists(athenzDomainPath())) return Optional.empty();
+ return Optional.ofNullable(configCurator.getData(athenzDomainPath()))
+ .filter(domain -> ! domain.isBlank())
+ .map(AthenzDomain::from);
+ }
+
/**
* Create necessary paths atomically for a new session.
*
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
index 65ebb38c2d0..97179e5234b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.tenant;
import com.yahoo.config.model.api.ContainerEndpoint;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.path.Path;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.transaction.CuratorOperations;
import com.yahoo.vespa.curator.transaction.CuratorTransaction;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
index 6500449e557..8e51ac424f9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
@@ -5,7 +5,7 @@ import com.yahoo.config.model.api.EndpointCertificateMetadata;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.path.Path;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.curator.Curator;
import com.yahoo.vespa.curator.transaction.CuratorOperations;
import com.yahoo.vespa.curator.transaction.CuratorTransaction;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
index 3ae678969eb..5f180bdaee1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
@@ -9,14 +9,12 @@ import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
/**
* Responsible for providing data from an application subtree in zookeeper.
* (i.e. /config/v2/tenants/x/session/&lt;session id for an application&gt;/).
*
- * Takes care of
- *
- *
* @author Tony Vaagenes
*/
public class ZKApplication {
@@ -84,6 +82,10 @@ public class ZKApplication {
return reader(data);
}
+ Optional<Reader> getOptionalDataReader(String path, String node) {
+ return Optional.ofNullable(getData(path, node)).map(data -> reader(data));
+ }
+
public String getData(String path, String node) {
try {
return zk.getData(getFullPath(path), node);
@@ -181,10 +183,9 @@ public class ZKApplication {
}
Reader getDataReader(String path) {
- final String data = getData(path);
- if (data == null) {
+ String data = getData(path);
+ if (data == null)
throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists");
- }
return reader(data);
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
index bcb958c4b58..c7ec2657996 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
@@ -139,13 +139,16 @@ public class ZKApplicationPackage implements ApplicationPackage {
@Override
public List<NamedReader> searchDefinitionContents() {
- List<NamedReader> ret = new ArrayList<>();
- for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) {
- if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) {
- ret.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd))));
- }
+ List<NamedReader> schemas = new ArrayList<>();
+ for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
+ if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX))
+ schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
}
- return ret;
+ for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) {
+ if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX))
+ schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd))));
+ }
+ return schemas;
}
@Override
@@ -176,7 +179,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
try {
return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def);
} catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e);
+ throw new IllegalArgumentException("Could not retrieve config definition " + def, e);
}
}
@@ -264,7 +267,10 @@ public class ZKApplicationPackage implements ApplicationPackage {
@Override
public Reader getRankingExpression(String name) {
- return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name);
+ Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name);
+ if (reader.isPresent())
+ return reader.get();
+ return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name);
}
@Override
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index 970dd49e865..0cd283ca62b 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -63,8 +63,7 @@
<component id="com.yahoo.vespa.service.manager.UnionMonitorManager" bundle="service-monitor" />
<component id="com.yahoo.vespa.service.model.ServiceMonitorImpl" bundle="service-monitor" />
<component id="com.yahoo.vespa.service.duper.DuperModelManager" bundle="service-monitor" />
- <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" />
- <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" />
+ <component id="com.yahoo.vespa.orchestrator.status.ZkStatusService" bundle="orchestrator" />
<component id="com.yahoo.vespa.orchestrator.controller.RetryingClusterControllerClientFactory" bundle="orchestrator" />
<component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" />
diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver
index bec206214f8..bded46dbebe 100755
--- a/configserver/src/main/sh/start-configserver
+++ b/configserver/src/main/sh/start-configserver
@@ -174,6 +174,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \
--add-opens=java.base/java.lang=ALL-UNNAMED \
--add-opens=java.base/java.net=ALL-UNNAMED \
--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \
+ -Djava.io.tmpdir=${VESPA_HOME}/tmp \
-Djava.library.path=${VESPA_HOME}/lib64 \
-Djava.awt.headless=true \
-Dsun.rmi.dgc.client.gcInterval=3600000 \
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index a963252d7ca..0e076d60d52 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -342,11 +342,10 @@ public class ApplicationRepositoryTest {
new MockTesterClient(),
actual);
deployApp(testAppLogServerWithContainer);
- Map<String, ?> context = Map.of("tenant", "test1",
- "application", "testapp",
- "instance", "default",
- "environment", "prod",
- "region", "default");
+ Map<String, ?> context = Map.of("applicationId", "test1.testapp.default",
+ "tenantName", "test1",
+ "app", "testapp.default",
+ "zone", "prod.default");
MockMetric expected = new MockMetric();
expected.set("deployment.prepareMillis", 0L, expected.createContext(context));
expected.set("deployment.activateMillis", 0L, expected.createContext(context));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 2a1254d0d8d..654d811a31f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.RegionName;
import com.yahoo.component.Version;
import com.yahoo.config.provision.Zone;
import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.core.VipStatusConfig;
import com.yahoo.container.handler.ClustersStatus;
import com.yahoo.container.handler.VipStatus;
import com.yahoo.container.jdisc.config.HealthMonitorConfig;
@@ -235,6 +236,7 @@ public class ConfigServerBootstrapTest {
private VipStatus createVipStatus(StateMonitor stateMonitor) {
return new VipStatus(new QrSearchersConfig.Builder().build(),
+ new VipStatusConfig.Builder().build(),
new ClustersStatus(),
stateMonitor);
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
index db687857dae..c835bb29ec0 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
@@ -15,70 +15,40 @@ public class MockTesterClient extends TesterClient {
@Override
public HttpResponse getStatus(String testerHostname, int port) {
- return new MockStatusResponse();
+ return new HttpResponse(200) {
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ outputStream.write("OK".getBytes(StandardCharsets.UTF_8));
+ }
+ };
}
@Override
public HttpResponse getLog(String testerHostname, int port, Long after) {
- return new MockLogResponse();
+ return new HttpResponse(200) {
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ outputStream.write("log".getBytes(StandardCharsets.UTF_8));
+ }
+ };
}
@Override
public HttpResponse startTests(String testerHostname, int port, String suite, byte[] config) {
- return new MockStartTestsResponse();
+ return new HttpResponse(200) {
+ @Override
+ public void render(OutputStream outputStream) { }
+ };
}
@Override
public HttpResponse isTesterReady(String testerHostname, int port) {
- return new MockTesterReadyResponse();
- }
-
- private static class MockStatusResponse extends HttpResponse {
-
- private MockStatusResponse() {
- super(200);
- }
-
- @Override
- public void render(OutputStream outputStream) throws IOException {
- outputStream.write("OK".getBytes(StandardCharsets.UTF_8));
- }
-
- }
-
- private static class MockLogResponse extends HttpResponse {
-
- private MockLogResponse() {
- super(200);
- }
-
- @Override
- public void render(OutputStream outputStream) throws IOException {
- outputStream.write("log".getBytes(StandardCharsets.UTF_8));
- }
-
- }
-
- private static class MockStartTestsResponse extends HttpResponse {
-
- private MockStartTestsResponse() {
- super(200);
- }
-
- @Override
- public void render(OutputStream outputStream) { }
-
- }
-
- private static class MockTesterReadyResponse extends HttpResponse {
-
- private MockTesterReadyResponse() {
- super(200);
- }
-
- @Override
- public void render(OutputStream outputStream) { }
-
+ return new HttpResponse(200) {
+ @Override
+ public void render(OutputStream outputStream) throws IOException {
+ outputStream.write("{ \"message\": \"OK\" } ".getBytes(StandardCharsets.UTF_8));
+ }
+ };
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
index 370e0bd3c0e..03c6bad79a8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server;
import com.yahoo.component.Version;
import com.yahoo.config.model.api.ContainerEndpoint;
import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.api.Provisioned;
import com.yahoo.config.model.application.provider.BaseDeployLogger;
import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.model.test.MockApplicationPackage;
@@ -20,6 +21,7 @@ import java.util.Set;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
@@ -45,6 +47,7 @@ public class ModelContextImplTest {
new StaticConfigDefinitionRepo(),
new MockFileRegistry(),
Optional.empty(),
+ new Provisioned(),
new ModelContextImpl.Properties(
ApplicationId.defaultId(),
true,
@@ -58,10 +61,12 @@ public class ModelContextImplTest {
false,
false,
flagSource,
- null),
+ null,
+ Optional.empty()),
+ Optional.empty(),
Optional.empty(),
- new Version(6),
- new Version(6));
+ new Version(7),
+ new Version(8));
assertTrue(context.applicationPackage() instanceof MockApplicationPackage);
assertFalse(context.hostProvisioner().isPresent());
assertFalse(context.permanentApplicationPackage().isPresent());
@@ -75,6 +80,13 @@ public class ModelContextImplTest {
assertFalse(context.properties().hostedVespa());
assertThat(context.properties().endpoints(), equalTo(endpoints));
assertThat(context.properties().isFirstTimeDeployment(), equalTo(false));
+
+ assertEquals(Optional.empty(), context.wantedDockerImageRepository());
+ assertEquals(new Version(7), context.modelVespaVersion());
+ assertEquals(new Version(8), context.wantedNodeVespaVersion());
+ assertEquals(1.0, context.properties().defaultTermwiseLimit(), 0.0);
+ assertEquals(1.0, context.properties().defaultTopKProbability(), 0.0);
+ assertFalse(context.properties().useAdaptiveDispatch());
}
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java
index f2ee7815df3..9a18570db2d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java
@@ -27,7 +27,6 @@ public class ServerCacheTest {
private static String configMd5 = "mymd5";
private static String configMd5_2 = "mymd5_2";
private static ConfigDefinition def = new ConfigDefinition("mypayload", new String[0]);
- private static ConfigDefinition def_2 = new ConfigDefinition("otherpayload", new String[0]);
private static ConfigDefinitionKey fooBarDefKey = new ConfigDefinitionKey("foo", "bar");
private static ConfigDefinitionKey fooBazDefKey = new ConfigDefinitionKey("foo", "baz");
@@ -49,9 +48,9 @@ public class ServerCacheTest {
cache = new ServerCache(new TestConfigDefinitionRepo(), userConfigDefinitionRepo);
- cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5);
- cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5);
- cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def_2.getCNode(), 2, false, configMd5_2), configMd5_2);
+ cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5);
+ cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5);
+ cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5_2), configMd5_2);
}
@Test
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index b7486dc7951..561422c1cf8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -55,7 +55,7 @@ public class SuperModelControllerTest {
ApplicationId app = ApplicationId.from(TenantName.from("a"),
ApplicationName.from("foo"), InstanceName.defaultName());
models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp))));
- SuperModel superModel = new SuperModel(models);
+ SuperModel superModel = new SuperModel(models, true);
handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
}
@@ -98,7 +98,7 @@ public class SuperModelControllerTest {
models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
- SuperModel superModel = new SuperModel(models);
+ SuperModel superModel = new SuperModel(models, true);
SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
han.getSuperModel().getConfig(lb);
@@ -126,7 +126,7 @@ public class SuperModelControllerTest {
models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
- SuperModel superModel = new SuperModel(models);
+ SuperModel superModel = new SuperModel(models, true);
SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
han.getSuperModel().getConfig(lb);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
index 405fff3e190..ad910c2afc2 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
@@ -4,14 +4,15 @@ package com.yahoo.vespa.config.server.application;
import com.yahoo.cloud.config.ModelConfig;
import com.yahoo.cloud.config.SlobroksConfig;
import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.component.Version;
import com.yahoo.config.SimpletypesConfig;
import com.yahoo.config.model.application.provider.FilesApplicationPackage;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ApplicationName;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.TenantName;
-import com.yahoo.component.Version;
import com.yahoo.jrt.Request;
+import com.yahoo.text.Utf8;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.GetConfigRequest;
@@ -33,12 +34,14 @@ import org.junit.Before;
import org.junit.Test;
import org.xml.sax.SAXException;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
import java.util.Optional;
import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -111,22 +114,27 @@ public class ApplicationTest {
}
@Test
- public void require_that_build_config_can_be_resolved() {
- List<String> payload = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)).getLegacyPayload();
- assertTrue(payload.get(1).contains("host"));
+ public void require_that_build_config_can_be_resolved() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE,
+ ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA))
+ .serialize(baos, CompressionType.UNCOMPRESSED);
+ assertTrue(baos.toString().startsWith("{\"vespaVersion\":\"1.0.0\",\"hosts\":[{\"name\":\"mytesthost\""));
}
@Test
- public void require_that_non_existent_fields_in_schema_is_skipped() {
+ public void require_that_non_existent_fields_in_schema_is_skipped() throws IOException {
// Ask for config without schema and check that we get correct default value back
- List<String> payload = handler.resolveConfig(createSimpleConfigRequest()).getLegacyPayload();
- assertThat(payload.get(0), is("boolval false"));
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ handler.resolveConfig(createSimpleConfigRequest()).serialize(baos, CompressionType.UNCOMPRESSED);;
+ assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(StandardCharsets.UTF_8));
// Ask for config with wrong schema
String[] schema = new String[1];
schema[0] = "boolval bool default=true"; // changed to be true, original is false
- payload = handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)).getLegacyPayload();
- assertThat(payload.size(), is(1));
- assertThat(payload.get(0), is("boolval true"));
+ baos = new ByteArrayOutputStream();
+ handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema))
+ .serialize(baos, CompressionType.UNCOMPRESSED);
+ assertEquals("{\"boolval\":true,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(Utf8.getCharset()));
}
@Test
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
index 71ae6955e56..1f034a92cbb 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
@@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName;
import com.yahoo.component.Version;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
import com.yahoo.vespa.config.server.ServerCache;
import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
import org.junit.Before;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
index beb8abb7be6..88d486cef87 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
@@ -2,15 +2,20 @@
package com.yahoo.vespa.config.server.application;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
import com.yahoo.vespa.applicationmodel.HostName;
import com.yahoo.vespa.orchestrator.Host;
import com.yahoo.vespa.orchestrator.Orchestrator;
import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
import com.yahoo.vespa.orchestrator.status.HostStatus;
+import java.time.Instant;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
@@ -22,7 +27,7 @@ import java.util.function.Function;
*/
public class OrchestratorMock implements Orchestrator {
- private final Set<HostName> suspendedHosts = new HashSet<>();
+ private final Map<HostName, HostInfo> hostInfos = new HashMap<>();
private final Set<ApplicationId> suspendedApplications = new HashSet<>();
@Override
@@ -32,12 +37,19 @@ public class OrchestratorMock implements Orchestrator {
@Override
public HostStatus getNodeStatus(HostName hostName) {
- return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+ HostInfo hostInfo = hostInfos.get(hostName);
+ return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status();
}
@Override
- public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
- return hostName -> Optional.of(getNodeStatus(hostName));
+ public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostname) {
+ HostInfo hostInfo = hostInfos.get(hostname);
+ return hostInfo == null ? HostInfo.createNoRemarks() : hostInfo;
+ }
+
+ @Override
+ public Function<HostName, Optional<HostInfo>> getHostResolver() {
+ return hostName -> Optional.of(hostInfos.getOrDefault(hostName, HostInfo.createNoRemarks()));
}
@Override
@@ -45,12 +57,12 @@ public class OrchestratorMock implements Orchestrator {
@Override
public void resume(HostName hostName) {
- suspendedHosts.remove(hostName);
+ hostInfos.remove(hostName);
}
@Override
public void suspend(HostName hostName) {
- suspendedHosts.add(hostName);
+ hostInfos.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH));
}
@Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index 32b704dd551..84987bce32e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -216,16 +216,32 @@ public class DeployTester {
return deployApp(applicationPath, vespaVersion, Instant.now());
}
+
/**
* Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet.
*/
- public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now) {
- PrepareParams.Builder paramsBuilder = new PrepareParams.Builder()
- .applicationId(applicationId)
- .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60)));
+ public PrepareResult deployApp(String applicationPath, String vespaVersion, String dockerImageRepository) {
+ PrepareParams.Builder paramsBuilder = new PrepareParams.Builder();
if (vespaVersion != null)
paramsBuilder.vespaVersion(vespaVersion);
+ return deployApp(applicationPath, Instant.now(), paramsBuilder.dockerImageRepository(dockerImageRepository));
+ }
+
+ /**
+ * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet.
+ */
+ public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now) {
+ return deployApp(applicationPath, now, new PrepareParams.Builder().vespaVersion(vespaVersion));
+ }
+
+ /**
+ * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet.
+ */
+ public PrepareResult deployApp(String applicationPath, Instant now, PrepareParams.Builder paramsBuilder) {
+ paramsBuilder.applicationId(applicationId)
+ .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60)));
+
return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), false, now);
}
@@ -283,8 +299,14 @@ public class DeployTester {
}
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
- return hostProvisioner.prepare(cluster, capacity, groups, logger);
+ return hostProvisioner.prepare(cluster, capacity.withGroups(groups), logger);
+ }
+
+ @Override
+ public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+ return hostProvisioner.prepare(cluster, capacity, logger);
}
@Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
index dd0c4eaf342..7e700b78bf7 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.config.server.configchange.RestartActions;
import com.yahoo.vespa.config.server.http.InvalidApplicationException;
import com.yahoo.vespa.config.server.http.v2.PrepareResult;
import com.yahoo.vespa.config.server.model.TestModelFactory;
+import com.yahoo.vespa.config.server.session.PrepareParams;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -87,6 +88,24 @@ public class HostedDeployTest {
}
@Test
+ public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException {
+ CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC());
+ DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig());
+ String dockerImageRepository = "docker.foo.com:4443/bar/baz";
+ tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder()
+ .vespaVersion("4.5.6")
+ .dockerImageRepository(dockerImageRepository)
+ .athenzDomain("foo"));
+
+ Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId());
+ assertTrue(deployment.isPresent());
+ deployment.get().activate();
+ assertEquals("4.5.6", ((Deployment) deployment.get()).session().getVespaVersion().toString());
+ assertEquals(dockerImageRepository, ((Deployment) deployment.get()).session().getDockerImageRepository().get());
+ assertEquals("foo", ((Deployment) deployment.get()).session().getAthenzDomain().get().value());
+ }
+
+ @Test
public void testDeployMultipleVersions() throws IOException {
List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")),
createHostedModelFactory(Version.fromString("6.2.0")),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java
index 9a371639f52..ef0a5f6113d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java
@@ -2,34 +2,25 @@
package com.yahoo.vespa.config.server.http;
import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
-import com.yahoo.text.StringUtilities;
import com.yahoo.vespa.config.ConfigPayload;
import com.yahoo.vespa.config.protocol.ConfigResponse;
-
import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
import org.junit.Test;
import java.io.IOException;
-import java.io.StringReader;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
/**
* @author Ulf Lilleengen
- * @since 5.1
*/
public class HttpConfigResponseTest {
@Test
public void require_that_response_is_created_from_config() throws IOException {
final long generation = 1L;
ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- // TODO: Hope to be able to remove this mess soon.
- DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
- InnerCNode targetDef = dParser.getTree();
- ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5");
+ ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5");
HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse);
assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java
index 3ae98c1b8f2..089b662b797 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java
@@ -2,28 +2,27 @@
package com.yahoo.vespa.config.server.http;
import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
+import com.yahoo.config.provision.ApplicationId;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.text.StringUtilities;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
-import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
-import com.yahoo.config.provision.ApplicationId;
-
+import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
import org.junit.Before;
import org.junit.Test;
+
import java.io.IOException;
-import java.io.StringReader;
import java.util.Collections;
import java.util.HashSet;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-import static com.yahoo.jdisc.http.HttpResponse.Status.*;
+import static com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST;
+import static com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
@@ -48,8 +47,7 @@ public class HttpGetConfigHandlerTest {
// Define config response for mock handler
final long generation = 1L;
ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- InnerCNode targetDef = getInnerCNode();
- mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+ mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET));
assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
}
@@ -75,16 +73,10 @@ public class HttpGetConfigHandlerTest {
public void require_that_nocache_property_works() throws IOException {
long generation = 1L;
ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- InnerCNode targetDef = getInnerCNode();
- mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+ mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true"));
HttpResponse response = handler.handle(request);
assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
}
- private InnerCNode getInnerCNode() {
- // TODO: Hope to be able to remove this mess soon.
- DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
- return dParser.getTree();
- }
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
index 9a326a18dd5..5b0bb7885d8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
@@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
/**
@@ -55,15 +56,27 @@ public class SessionHandlerTest {
public static final String hostname = "foo";
public static final int port = 1337;
- public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath, InputStream data) {
- return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method, data);
+
+ public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+ Cmd cmd, Long id, String subPath, InputStream data, Map<String, String> properties) {
+ return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" +
+ cmd.toString() + subPath, method, data, properties);
+ }
+
+ public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+ Cmd cmd, Long id, String subPath, InputStream data) {
+ return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" +
+ cmd.toString() + subPath, method, data);
}
- public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath) {
- return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method);
+ public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+ Cmd cmd, Long id, String subPath) {
+ return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" +
+ cmd.toString() + subPath, method);
}
- public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id) {
+ public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+ Cmd cmd, Long id) {
return createTestRequest(path, method, cmd, id, "");
}
@@ -88,6 +101,7 @@ public class SessionHandlerTest {
private ConfigChangeActions actions = new ConfigChangeActions();
private long createTime = System.currentTimeMillis() / 1000;
private ApplicationId applicationId;
+ private Optional<String> dockerImageRepository;
public MockSession(long id, ApplicationPackage app) {
this(id, app, new InMemoryFlagSource());
@@ -115,6 +129,7 @@ public class SessionHandlerTest {
@Override
public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath, Instant now) {
status = Session.Status.PREPARE;
+ this.dockerImageRepository = params.dockerImageRepository();
if (doVerboseLogging) {
logger.log(LogLevel.DEBUG, "debuglog");
}
@@ -158,6 +173,10 @@ public class SessionHandlerTest {
@Override
public void delete(NestedTransaction transaction) { }
+ @Override
+ public Optional<String> getDockerImageRepository() {
+ return dockerImageRepository;
+ }
}
public enum Cmd {
@@ -218,11 +237,17 @@ public class SessionHandlerTest {
public Collection<HostSpec> lastHosts;
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
throw new UnsupportedOperationException();
}
@Override
+ public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
activated = true;
lastApplicationId = application;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 67677822317..70f66cf8fde 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -28,12 +28,12 @@ import com.yahoo.vespa.config.server.session.PrepareParams;
import com.yahoo.vespa.config.server.tenant.Tenant;
import com.yahoo.vespa.config.server.tenant.TenantBuilder;
import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import javax.ws.rs.client.Client;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets;
import java.time.Clock;
import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
+import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -58,11 +59,16 @@ public class ApplicationHandlerTest {
private static File testApp = new File("src/test/apps/app");
- private ListApplicationsHandler listApplicationsHandler;
private final static TenantName mytenantName = TenantName.from("mytenant");
private final static TenantName foobar = TenantName.from("foobar");
- private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build();
-
+ private final static ApplicationId myTenantApplicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build();
+ private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(TenantName.defaultName()).build();
+ private final static MockTesterClient testerClient = new MockTesterClient();
+ private final static NullMetric metric = new NullMetric();
+ private final static ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder());
+ private static final MockLogRetriever logRetriever = new MockLogRetriever();
+
+ private TestComponentRegistry componentRegistry;
private TenantRepository tenantRepository;
private ApplicationRepository applicationRepository;
private SessionHandlerTest.MockProvisioner provisioner;
@@ -71,27 +77,30 @@ public class ApplicationHandlerTest {
@Before
public void setup() {
- TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build();
+ componentRegistry = new TestComponentRegistry.Builder().build();
tenantRepository = new TenantRepository(componentRegistry, false);
- tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName));
- tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar));
provisioner = new SessionHandlerTest.MockProvisioner();
orchestrator = new OrchestratorMock();
applicationRepository = new ApplicationRepository(tenantRepository,
provisioner,
orchestrator,
- new ConfigserverConfig(new ConfigserverConfig.Builder()),
- new MockLogRetriever(),
+ configserverConfig,
+ logRetriever,
Clock.systemUTC(),
- new MockTesterClient(),
- new NullMetric());
- listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(),
- tenantRepository,
- Zone.defaultZone());
+ testerClient,
+ metric);
+ }
+
+ @After
+ public void shutdown() {
+ tenantRepository.close();
}
@Test
public void testDelete() throws Exception {
+ tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar));
+ tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName));
+
{
applicationRepository.deploy(testApp, prepareParams(applicationId));
Tenant mytenant = tenantRepository.getTenant(applicationId.tenant());
@@ -132,7 +141,7 @@ public class ApplicationHandlerTest {
@Test
public void testDeleteNonExistent() throws Exception {
- deleteAndAssertResponse(applicationId,
+ deleteAndAssertResponse(myTenantApplicationId,
Zone.defaultZone(),
Response.Status.NOT_FOUND,
HttpErrorResponse.errorCodes.NOT_FOUND,
@@ -180,10 +189,10 @@ public class ApplicationHandlerTest {
InfraDeployerProvider.empty(),
new ConfigConvergenceChecker(stateApiFactory),
mockHttpProxy,
- new ConfigserverConfig(new ConfigserverConfig.Builder()),
- new OrchestratorMock(),
- new MockTesterClient(),
- new NullMetric());
+ configserverConfig,
+ orchestrator,
+ testerClient,
+ metric);
ApplicationHandler mockHandler = createApplicationHandler(applicationRepository);
when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1")))
.thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));
@@ -204,16 +213,15 @@ public class ApplicationHandlerTest {
HttpResponse response = fileDistributionStatus(applicationId, zone);
assertEquals(200, response.getStatus());
- SessionHandlerTest.getRenderedString(response);
assertEquals("{\"hosts\":[{\"hostname\":\"mytesthost\",\"status\":\"UNKNOWN\",\"message\":\"error: Connection error(104)\",\"fileReferences\":[]}],\"status\":\"UNKNOWN\"}",
- SessionHandlerTest.getRenderedString(response));
+ getRenderedString(response));
// 404 for unknown application
- ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant(mytenantName).build();
+ ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant("default").build();
HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone);
assertEquals(404, responseForUnknown.getStatus());
- assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'mytenant.unknown'\"}",
- SessionHandlerTest.getRenderedString(responseForUnknown));
+ assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'default.unknown'\"}",
+ getRenderedString(responseForUnknown));
}
@Test
@@ -225,9 +233,7 @@ public class ApplicationHandlerTest {
HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
assertEquals(200, response.getStatus());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- response.render(baos);
- assertEquals("log line", baos.toString());
+ assertEquals("log line", getRenderedString(response));
}
@Test
@@ -235,13 +241,9 @@ public class ApplicationHandlerTest {
applicationRepository.deploy(testApp, prepareParams(applicationId));
String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status";
ApplicationHandler mockHandler = createApplicationHandler();
-
HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
assertEquals(200, response.getStatus());
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- response.render(baos);
- assertEquals("OK", baos.toString());
+ assertEquals("OK", getRenderedString(response));
}
@Test
@@ -252,10 +254,7 @@ public class ApplicationHandlerTest {
HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
assertEquals(200, response.getStatus());
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- response.render(baos);
- assertEquals("log", baos.toString());
+ assertEquals("log", getRenderedString(response));
}
@Test
@@ -345,10 +344,13 @@ public class ApplicationHandlerTest {
"/environment/" + zone.environment().value() +
"/region/" + zone.region().value() +
"/instance/" + applicationId.instance().value() + "\"]";
+ ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(),
+ tenantRepository,
+ Zone.defaultZone());
ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/",
- Response.Status.OK,
- expected,
- com.yahoo.jdisc.http.HttpRequest.Method.GET);
+ Response.Status.OK,
+ expected,
+ com.yahoo.jdisc.http.HttpRequest.Method.GET);
}
private void restart(ApplicationId application, Zone zone) throws IOException {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java
index fb09aa99039..b72785876bc 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java
@@ -1,37 +1,36 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.http.v2;
-import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
-import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
-import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.Collections;
-import java.util.HashSet;
-
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.config.server.TestComponentRegistry;
-import com.yahoo.vespa.config.server.http.HttpErrorResponse;
-import com.yahoo.vespa.config.server.tenant.TenantBuilder;
-import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import org.junit.Before;
-import org.junit.Test;
import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.TenantName;
import com.yahoo.container.jdisc.HttpRequest;
import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.text.StringUtilities;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
-import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.config.server.TestComponentRegistry;
import com.yahoo.vespa.config.server.http.HandlerTest;
import com.yahoo.vespa.config.server.http.HttpConfigRequest;
+import com.yahoo.vespa.config.server.http.HttpErrorResponse;
import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
+import com.yahoo.vespa.config.server.tenant.TenantBuilder;
+import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+
+import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
+import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
+import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
public class HttpGetConfigHandlerTest {
@@ -58,9 +57,8 @@ public class HttpGetConfigHandlerTest {
// Define config response for mock handler
final long generation = 1L;
ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- InnerCNode targetDef = getInnerCNode();
mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(),
- SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+ SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET));
assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING));
}
@@ -71,11 +69,10 @@ public class HttpGetConfigHandlerTest {
"/application/myapplication/environment/staging/region/myregion/instance/myinstance/foo.bar/myid";
final long generation = 1L;
ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- InnerCNode targetDef = getInnerCNode();
mockRequestHandler.responses.put(new ApplicationId.Builder()
.tenant(tenant)
.applicationName("myapplication").instanceName("myinstance").build(),
- SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+ SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET));
assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING));
}
@@ -121,18 +118,11 @@ public class HttpGetConfigHandlerTest {
public void require_that_nocache_property_works() throws IOException {
long generation = 1L;
ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
- InnerCNode targetDef = getInnerCNode();
mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(),
- SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+ SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true"));
HttpResponse response = handler.handle(request);
assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING));
}
- private InnerCNode getInnerCNode() {
- // TODO: Hope to be able to remove this mess soon.
- DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
- return dParser.getTree();
- }
-
}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
index 028f5f9eb8c..11cbdb03ccf 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
@@ -44,6 +44,7 @@ import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
@@ -90,7 +91,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
@Test
public void require_error_when_session_id_does_not_exist() throws Exception {
// No session with this id exists
- HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 9999L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 9999L);
assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "Session 9999 was not found");
}
@@ -120,8 +121,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_that_activate_url_is_returned_on_success() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- HttpResponse response = createHandler().handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertThat(session.getStatus(), is(Session.Status.PREPARE));
assertNotNull(response);
assertThat(response.getStatus(), is(OK));
@@ -155,13 +155,11 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_get_response_activate_url_on_ok() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- SessionHandler sessHandler = createHandler();
- sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ request(HttpRequest.Method.PUT, 1L);
session.setStatus(Session.Status.PREPARE);
SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session);
zooKeeperClient.writeStatus(Session.Status.PREPARE);
- HttpResponse getResponse = sessHandler.handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L));
+ HttpResponse getResponse = request(HttpRequest.Method.GET, 1L);
assertResponseContains(getResponse, "\"activate\":\"http://foo:1337" + pathPrefix +
"1/active\",\"message\":\"Session 1" + preparedMessage);
}
@@ -170,19 +168,16 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_get_response_error_on_not_prepared() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- SessionHandler sessHandler = createHandler();
session.setStatus(Session.Status.NEW);
SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session);
zooKeeperClient.writeStatus(Session.Status.NEW);
- HttpResponse getResponse = sessHandler.handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L));
+ HttpResponse getResponse = request(HttpRequest.Method.GET, 1L);
assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST,
HttpErrorResponse.errorCodes.BAD_REQUEST,
"Session not prepared: 1");
session.setStatus(Session.Status.ACTIVATE);
zooKeeperClient.writeStatus(Session.Status.ACTIVATE);
- getResponse = sessHandler.handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L));
+ getResponse = request(HttpRequest.Method.GET, 1L);
assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST,
HttpErrorResponse.errorCodes.BAD_REQUEST,
"Session is active: 1");
@@ -193,9 +188,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
session.setStatus(Session.Status.ACTIVATE);
- SessionHandler sessionHandler = createHandler();
- HttpResponse putResponse = sessionHandler.handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse putResponse = request(HttpRequest.Method.PUT, 1L);
assertHttpStatusCodeErrorCodeAndMessage(putResponse, BAD_REQUEST,
HttpErrorResponse.errorCodes.BAD_REQUEST,
"Session is active: 1");
@@ -205,9 +198,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_get_response_error_when_session_id_does_not_exist() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- SessionHandler sessHandler = createHandler();
- HttpResponse getResponse = sessHandler.handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 9999L));
+ HttpResponse getResponse = request(HttpRequest.Method.GET, 9999L);
assertHttpStatusCodeErrorCodeAndMessage(getResponse, NOT_FOUND,
HttpErrorResponse.errorCodes.NOT_FOUND,
"Session 9999 was not found");
@@ -217,8 +208,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_that_tenant_is_in_response() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- HttpResponse response = createHandler().handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertNotNull(response);
assertThat(response.getStatus(), is(OK));
assertThat(session.getStatus(), is(Session.Status.PREPARE));
@@ -242,8 +232,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
localRepoDefault.addSession(session);
pathPrefix = "/application/v2/tenant/" + defaultTenant + "/session/";
- HttpResponse response = handler.handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId));
+ HttpResponse response = request(HttpRequest.Method.PUT, sessionId);
assertNotNull(response);
assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK));
assertThat(session.getStatus(), is(Session.Status.PREPARE));
@@ -274,8 +263,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_that_config_change_actions_are_in_response() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- HttpResponse response = createHandler().handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertResponseContains(response, "\"configChangeActions\":{\"restart\":[],\"refeed\":[]}");
}
@@ -289,8 +277,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
new MockRefeedAction("change-id", false, "other change", services, "test")));
MockSession session = new MockSession(1, null, actions);
localRepo.addSession(session);
- HttpResponse response = createHandler().handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertResponseContains(response,
"Change(s) between active and new application that require restart:\\nIn cluster 'foo' of type 'bar");
assertResponseContains(response,
@@ -301,8 +288,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
public void require_that_config_change_actions_are_not_logged_if_not_existing() throws Exception {
MockSession session = new MockSession(1, null);
localRepo.addSession(session);
- HttpResponse response = createHandler().handle(
- SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertResponseNotContains(response, "Change(s) between active and new application that require restart");
assertResponseNotContains(response, "Change(s) between active and new application that require re-feed");
}
@@ -312,8 +298,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
String message = "Internal error";
SessionThrowingException session = new SessionThrowingException(new OutOfCapacityException(message));
localRepo.addSession(session);
- HttpResponse response = createHandler()
- .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertEquals(400, response.getStatus());
Slime data = getData(response);
assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.OUT_OF_CAPACITY.name()));
@@ -325,8 +310,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
String message = "No nodes available";
SessionThrowingException session = new SessionThrowingException(new NullPointerException(message));
localRepo.addSession(session);
- HttpResponse response = createHandler()
- .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertEquals(500, response.getStatus());
Slime data = getData(response);
assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR.name()));
@@ -339,14 +323,22 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
SessionThrowingException session =
new SessionThrowingException(new ApplicationLockException(new UncheckedTimeoutException(message)));
localRepo.addSession(session);
- HttpResponse response = createHandler()
- .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+ HttpResponse response = request(HttpRequest.Method.PUT, 1L);
assertEquals(500, response.getStatus());
Slime data = getData(response);
assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.APPLICATION_LOCK_FAILURE.name()));
assertThat(data.get().field("message").asString(), is(message));
}
+ @Test
+ public void test_docker_image_repository() {
+ MockSession session = new MockSession(1, null);
+ localRepo.addSession(session);
+ String dockerImageRepository = "https://foo.bar.com:4443/baz";
+ request(HttpRequest.Method.PUT, 1L, Map.of("dockerImageRepository", dockerImageRepository));
+ assertEquals(dockerImageRepository, localRepo.getSession(1).getDockerImageRepository().get());
+ }
+
private Slime getData(HttpResponse response) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
response.render(baos);
@@ -374,6 +366,14 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
}
+ private HttpResponse request(HttpRequest.Method put, long l) {
+ return request(put, l, Map.of());
+ }
+
+ private HttpResponse request(HttpRequest.Method put, long l, Map<String, String> requestParameters) {
+ return createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, put, Cmd.PREPARED, l, "", null, requestParameters));
+ }
+
public static class SessionThrowingException extends LocalSession {
private final RuntimeException exception;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java
index 0894e38ce09..3f67d8e2cac 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java
@@ -14,13 +14,15 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
/**
@@ -29,11 +31,13 @@ import static org.junit.Assert.*;
public class ClusterMetricsRetrieverTest {
@Rule
- public final WireMockRule wireMock = new WireMockRule(options().port(8080), true);
+ public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true);
@Test
public void testMetricAggregation() throws IOException {
- List<URI> hosts = List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2"), URI.create("http://localhost:8080/3"));
+ List<URI> hosts = Stream.of(1, 2, 3)
+ .map(item -> URI.create("http://localhost:" + wireMock.port() + "/" + item))
+ .collect(Collectors.toList());
stubFor(get(urlEqualTo("/1"))
.willReturn(aResponse()
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java
index 9a7cb72804f..f75d11a145f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java
@@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeFalse;
@@ -53,7 +54,7 @@ public class LbServicesProducerTest {
private static final Set<ContainerEndpoint> endpoints = Set.of(
new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2"))
);
- private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+ private InMemoryFlagSource flagSource = new InMemoryFlagSource();
private final boolean useGlobalServiceId;
@Parameterized.Parameters
@@ -141,6 +142,23 @@ public class LbServicesProducerTest {
assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2));
}
+
+ @Test
+ public void testRoutingConfigForTesterApplication() throws IOException, SAXException {
+ assumeFalse(useGlobalServiceId);
+
+ Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder());
+ LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel);
+ LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName);
+ assertThat(services.servicealiases().size(), is(1));
+ assertThat(services.endpointaliases().size(), is(2));
+
+ // No config for tester application
+ assertNull(getLbServicesConfig(Zone.defaultZone(), testModel)
+ .tenants("foo")
+ .applications("baz:prod:default:custom-t"));
+ }
+
private Map<TenantName, Set<ApplicationInfo>> randomizeApplications(Map<TenantName, Set<ApplicationInfo>> testModel, int seed) {
Map<TenantName, Set<ApplicationInfo>> randomizedApplications = new LinkedHashMap<>();
List<TenantName> keys = new ArrayList<>(testModel.keySet());
@@ -166,7 +184,7 @@ public class LbServicesProducerTest {
Set<ApplicationInfo> aMap = new LinkedHashSet<>();
ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build();
ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build();
- ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").build();
+ ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").instanceName("custom-t").build(); // tester app
aMap.add(createApplication(fooApp, deploystateBuilder));
aMap.add(createApplication(barApp, deploystateBuilder));
aMap.add(createApplication(bazApp, deploystateBuilder));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java
index ed089109759..607a2dca6c6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java
@@ -57,12 +57,14 @@ public class ZKMetricUpdaterTest {
assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_LATENCY_MAX), equalTo(1234L));
assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_OUTSTANDING_REQUESTS), equalTo(12L));
assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_ZNODES), equalTo(4L));
+
+ updater.shutdown();
}
private ZKMetricUpdater buildUpdater() {
ZookeeperServerConfig zkServerConfig = new ZookeeperServerConfig(
new ZookeeperServerConfig.Builder().clientPort(serverPort).myid(12345));
- return new ZKMetricUpdater(zkServerConfig, 0, -1);
+ return new ZKMetricUpdater(zkServerConfig, 0, 100000);
}
private void setupTcpServer(Supplier<String> reportProvider) throws IOException {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
index 086dfa5d0d3..0b33de2a42c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
@@ -1,18 +1,15 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.rpc;
-import com.google.common.base.Joiner;
import com.yahoo.cloud.config.ConfigserverConfig;
import com.yahoo.cloud.config.LbServicesConfig;
import com.yahoo.cloud.config.SentinelConfig;
+import com.yahoo.component.Version;
import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.TenantName;
-import com.yahoo.component.Version;
import com.yahoo.jrt.Request;
-import com.yahoo.config.provision.ApplicationId;
import com.yahoo.vespa.config.ConfigKey;
import com.yahoo.vespa.config.ConfigPayload;
import com.yahoo.vespa.config.ConfigPayloadApplier;
@@ -24,12 +21,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest;
import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
import com.yahoo.vespa.config.protocol.Trace;
-import com.yahoo.vespa.config.server.application.ApplicationSet;
import com.yahoo.vespa.config.server.ServerCache;
import com.yahoo.vespa.config.server.application.Application;
+import com.yahoo.vespa.config.server.application.ApplicationSet;
import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
import com.yahoo.vespa.config.util.ConfigUtils;
-
import com.yahoo.vespa.model.VespaModel;
import org.junit.Rule;
import org.junit.Test;
@@ -37,11 +33,14 @@ import org.junit.rules.TemporaryFolder;
import org.xml.sax.SAXException;
import java.io.IOException;
-import java.io.StringReader;
import java.util.Optional;
import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
/**
* @author Ulf Lilleengen
@@ -161,11 +160,7 @@ public class RpcServerTest {
builder.intval(123);
SimpletypesConfig responseConfig = new SimpletypesConfig(builder);
ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig);
- InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME,
- new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA)))
- .getTree();
return SlimeConfigResponse.fromConfigPayload(responsePayload,
- targetDef,
3L,
true, /* internalRedeploy */
ConfigUtils.getMd5(responsePayload));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index 40115170b69..a3b0f3ec44a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -342,11 +342,17 @@ public class SessionPreparerTest {
private static class FailWithTransientExceptionProvisioner implements Provisioner {
@Override
+ @Deprecated // TODO: Remove after April 2020
public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception"));
}
@Override
+ public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+ throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception"));
+ }
+
+ @Override
public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { }
@Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
index f2c6aac2bda..9c7da7134e6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provision.HostSpec;
import com.yahoo.config.provision.NodeFlavors;
@@ -39,7 +40,8 @@ public class ZKApplicationPackageTest {
private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME);
private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts(
Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty(),
- Optional.of(com.yahoo.component.Version.fromString("6.0.1")))));
+ Optional.of(Version.fromString("6.0.1")), Optional.empty(),
+ Optional.empty(), Optional.of(DockerImage.fromString("docker repo")))));
private ConfigCurator configCurator;
@@ -59,7 +61,7 @@ public class ZKApplicationPackageTest {
assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches());
assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches());
DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build();
- assertEquals(deployState.getSearchDefinitions().size(), 5);
+ assertEquals(deployState.getSchemas().size(), 5);
assertEquals(zkApp.searchDefinitionContents().size(), 5);
assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n");
assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3);
@@ -80,6 +82,8 @@ public class ZKApplicationPackageTest {
assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS))));
assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR));
assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString());
+ // TODO: Enable when dockerImageRepo is written to zk
+ //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get());
assertTrue(zkApp.getDeployment().isPresent());
assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get());
}
@@ -87,7 +91,7 @@ public class ZKApplicationPackageTest {
private void feed(ConfigCurator zk, File dirToFeed) throws IOException {
assertTrue(dirToFeed.isDirectory());
zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true);
- String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}";
+ String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}";
zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData);
zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles");
zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS));