diff options
author | HÃ¥kon Hallingstad <hakon@verizonmedia.com> | 2020-04-20 10:36:33 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-20 10:36:33 +0200 |
commit | b105eead1fbbcefbb85bc962749f2a12fa660bbe (patch) | |
tree | 7a3c996c00b854066d32608a002335715fb98c96 /configserver | |
parent | f61f6c701dc91e839b865f158a6da56ff166def7 (diff) | |
parent | 9ab0ef70e9ed4f422df67603f26bcb0c7918fdc4 (diff) |
Merge branch 'master' into hakonhall/remove-use-bucket-space-metric-feature-flag
Diffstat (limited to 'configserver')
60 files changed, 766 insertions, 484 deletions
diff --git a/configserver/pom.xml b/configserver/pom.xml index 1e242988327..8cd1b4b4254 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -215,11 +215,6 @@ <artifactId>jersey-proxy-client</artifactId> </dependency> <dependency> - <groupId>net.jpountz.lz4</groupId> - <artifactId>lz4</artifactId> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 92e7bf0300b..95c8733b540 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -314,8 +314,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye tenant.getLocalSessionRepo().addSession(newSession); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, - false /* don't validate as this is already deployed */, newSession.getVespaVersion(), - bootstrap)); + false /* don't validate as this is already deployed */, bootstrap)); } @Override @@ -893,11 +892,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public void close() { metric.set(name, Duration.between(start, clock.instant()).toMillis(), - metric.createContext(Map.of("tenant", id.tenant().value(), - "application", id.application().value(), - "instance", id.instance().value(), - "environment", environment, - "region", region))); + metric.createContext(Map.of("applicationId", id.toFullString(), + "tenantName", id.tenant().value(), + "app", id.application().value() + "." + id.instance().value(), + "zone", environment + "." + region))); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index b2a10f4bb21..2a426e4cccc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -79,7 +79,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable // For testing only ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState, - StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) { + StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) { this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode); } @@ -231,20 +231,17 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable new DaemonThreadFactory("redeploy apps")); // Keep track of deployment per application Map<ApplicationId, Future<?>> futures = new HashMap<>(); - Set<ApplicationId> failedDeployments = new HashSet<>(); - - for (ApplicationId appId : applicationIds) { - Optional<Deployment> deploymentOptional = applicationRepository.deployFromLocalActive(appId, true /* bootstrap */); - if (deploymentOptional.isEmpty()) continue; - - futures.put(appId, executor.submit(deploymentOptional.get()::activate)); - } + applicationIds.forEach(applicationId -> futures.put(applicationId, executor.submit(() -> { + applicationRepository.deployFromLocalActive(applicationId, true /* bootstrap */) + .ifPresent(Deployment::activate); + }))); + Set<ApplicationId> failedDeployments = new HashSet<>(); for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) { - ApplicationId app = f.getKey(); try { f.getValue().get(); } catch (ExecutionException e) { + ApplicationId app = f.getKey(); if (e.getCause() instanceof TransientException) { log.log(LogLevel.INFO, "Redeploying " + app + " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index ea835206b7c..d46c551fa7e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Zone; +import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; @@ -18,14 +19,19 @@ import com.yahoo.vespa.flags.FlagSource; import java.time.Instant; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; /** * Provides a SuperModel - a model of all application instances, and makes it stays * up to date as applications are added, redeployed, and removed. */ public class SuperModelManager implements SuperModelProvider { + private static final Logger logger = Logger.getLogger(SuperModelManager.class.getName()); + private final Zone zone; private final Object monitor = new Object(); @@ -38,6 +44,9 @@ public class SuperModelManager implements SuperModelProvider { private final long masterGeneration; // ConfigserverConfig's generation private final GenerationCounter generationCounter; + // The initial set of applications to be deployed on bootstrap. + private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty(); + @Inject public SuperModelManager(ConfigserverConfig configserverConfig, NodeFlavors nodeFlavors, @@ -75,6 +84,10 @@ public class SuperModelManager implements SuperModelProvider { listeners.add(listener); SuperModel superModel = superModelConfigProvider.getSuperModel(); superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application)); + + if (superModel.isComplete()) { + listener.notifyOfCompleteness(superModel); + } } } @@ -89,28 +102,37 @@ public class SuperModelManager implements SuperModelProvider { .getForVersionOrLatest(Optional.empty(), Instant.now()) .toApplicationInfo(); - SuperModel newSuperModel = this.superModelConfigProvider - .getSuperModel() + SuperModel newSuperModel = superModelConfigProvider.getSuperModel() .cloneAndSetApplication(applicationInfo); + generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); - listeners.stream().forEach(listener -> - listener.applicationActivated(newSuperModel, applicationInfo)); + listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo)); } } public void applicationRemoved(ApplicationId applicationId) { synchronized (monitor) { + bootstrapApplicationSet.ifPresent(set -> set.remove(applicationId)); + SuperModel newSuperModel = this.superModelConfigProvider .getSuperModel() .cloneAndRemoveApplication(applicationId); generationCounter.increment(); makeNewSuperModelConfigProvider(newSuperModel); - listeners.stream().forEach(listener -> - listener.applicationRemoved(newSuperModel, applicationId)); + listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId)); } } + public void markAsComplete() { + // Invoked on component graph bootstrap (even before ConfigServerBootstrap), + // there is no need to bump generation counter. + logger.log(LogLevel.INFO, "Super model is complete"); + SuperModel newSuperModel = getSuperModel().cloneAsComplete(); + superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); + listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); + } + private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) { generation = masterGeneration + generationCounter.get(); superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 520972f2fcf..51c5be1b1f0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -125,5 +125,7 @@ public class SuperModelRequestHandler implements RequestHandler { public void enable() { enabled = true; + superModelManager.markAsComplete(); + updateHandler(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 7ec0f49ed60..7ff5d41485d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -21,10 +21,8 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import java.time.Duration; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.logging.Logger; @@ -49,7 +47,6 @@ public class TenantApplications { private final Path locksPath; private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; - private final Map<ApplicationId, Lock> locks; private final Executor zkWatcherExecutor; private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant, @@ -57,7 +54,6 @@ public class TenantApplications { this.curator = curator; this.applicationsPath = TenantRepository.getApplicationsPath(tenant); this.locksPath = TenantRepository.getLocksPath(tenant); - this.locks = new ConcurrentHashMap<>(2); this.reloadHandler = reloadHandler; this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor); @@ -148,10 +144,7 @@ public class TenantApplications { /** Returns the lock for changing the session status of the given application. */ public Lock lock(ApplicationId id) { - curator.create(lockPath(id)); - Lock lock = locks.computeIfAbsent(id, __ -> new Lock(lockPath(id).getAbsolute(), curator)); - lock.acquire(Duration.ofMinutes(1)); // These locks shouldn't be held for very long. - return lock; + return curator.lock(lockPath(id), Duration.ofMinutes(1)); // These locks shouldn't be held for very long. } private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 89d7c349d6b..4bd5cf30cc6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -4,10 +4,9 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -25,7 +24,6 @@ import com.yahoo.vespa.curator.Lock; import java.time.Clock; import java.time.Duration; -import java.time.Instant; import java.util.Optional; import java.util.logging.Logger; @@ -49,13 +47,19 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private final Duration timeout; private final Clock clock; private final DeployLogger logger = new SilentDeployLogger(); - + + /** The repository part of docker image this application should run on. Version is separate from image repo */ + Optional<String> dockerImageRepository; + /** The Vespa version this application should run on */ private final Version version; /** True if this deployment is done to bootstrap the config server */ private final boolean isBootstrap; + /** The (optional) Athenz domain this application should use */ + private final Optional<AthenzDomain> athenzDomain; + private boolean prepared = false; /** Whether this model should be validated (only takes effect if prepared=false) */ @@ -64,9 +68,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private boolean ignoreSessionStaleFailure = false; private Deployment(LocalSession session, ApplicationRepository applicationRepository, - Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean prepared, boolean validate, Version version, - boolean isBootstrap) { + Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, + Clock clock, boolean prepared, boolean validate, boolean isBootstrap) { this.session = session; this.applicationRepository = applicationRepository; this.hostProvisioner = hostProvisioner; @@ -75,23 +78,24 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.clock = clock; this.prepared = prepared; this.validate = validate; - this.version = version; + this.dockerImageRepository = session.getDockerImageRepository(); + this.version = session.getVespaVersion(); this.isBootstrap = isBootstrap; + this.athenzDomain = session.getAthenzDomain(); } public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, - Duration timeout, Clock clock, boolean validate, Version version, - boolean isBootstrap) { - return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, false, validate, version, isBootstrap); + Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { + return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false, + validate, isBootstrap); } public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository, Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout, Clock clock, boolean isBootstrap) { return new Deployment(session, applicationRepository, hostProvisioner, tenant, - timeout, clock, true, true, session.getVespaVersion(), isBootstrap); + timeout, clock, true, true, isBootstrap); } public void setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) { @@ -105,16 +109,14 @@ public class Deployment implements com.yahoo.config.provision.Deployment { try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.prepareMillis")) { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - session.prepare(logger, - new PrepareParams.Builder().applicationId(session.getApplicationId()) - .timeoutBudget(timeoutBudget) - .ignoreValidationErrors(!validate) - .vespaVersion(version.toString()) - .isBootstrap(isBootstrap) - .build(), - Optional.empty(), - tenant.getPath(), - clock.instant()); + PrepareParams.Builder params = new PrepareParams.Builder().applicationId(session.getApplicationId()) + .timeoutBudget(timeoutBudget) + .ignoreValidationErrors(!validate) + .vespaVersion(version.toString()) + .isBootstrap(isBootstrap); + dockerImageRepository.ifPresent(params::dockerImageRepository); + athenzDomain.ifPresent(params::athenzDomain); + session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant()); this.prepared = true; } } @@ -129,10 +131,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); ApplicationId applicationId = session.getApplicationId(); + LocalSession previousActiveSession; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { validateSessionStatus(session); NestedTransaction transaction = new NestedTransaction(); - transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure)); + previousActiveSession = applicationRepository.getActiveSession(applicationId); + transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure)); hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); transaction.commit(); } @@ -147,8 +151,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + - (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") + + (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + + (previousActiveSession != null ? ". Activated session based on previous active session " + previousActiveSession.getSessionId() : "") + ". File references used: " + applicationRepository.getFileReferences(applicationId)); } } @@ -166,14 +171,13 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Exposes the session of this for testing only */ public LocalSession session() { return session; } - private long validateSessionStatus(LocalSession localSession) { + private void validateSessionStatus(LocalSession localSession) { long sessionId = localSession.getSessionId(); if (Session.Status.NEW.equals(localSession.getStatus())) { throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is not prepared"); } else if (Session.Status.ACTIVATE.equals(localSession.getStatus())) { throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is already active"); } - return sessionId; } private Transaction deactivateCurrentActivateNew(LocalSession active, LocalSession prepared, boolean ignoreStaleSessionFailure) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 23bdf972a06..400460d2ce6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -12,8 +12,10 @@ import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.TlsSecrets; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.FetchVector; @@ -40,9 +42,12 @@ public class ModelContextImpl implements ModelContext { private final ConfigDefinitionRepo configDefinitionRepo; private final FileRegistry fileRegistry; private final Optional<HostProvisioner> hostProvisioner; + private final Provisioned provisioned; private final ModelContext.Properties properties; private final Optional<File> appDir; + private final Optional<String> wantedDockerImageRepository; + /** The version of Vespa we are building a model for */ private final Version modelVespaVersion; @@ -62,8 +67,10 @@ public class ModelContextImpl implements ModelContext { ConfigDefinitionRepo configDefinitionRepo, FileRegistry fileRegistry, Optional<HostProvisioner> hostProvisioner, + Provisioned provisioned, ModelContext.Properties properties, Optional<File> appDir, + Optional<String> wantedDockerImageRepository, Version modelVespaVersion, Version wantedNodeVespaVersion) { this.applicationPackage = applicationPackage; @@ -73,8 +80,10 @@ public class ModelContextImpl implements ModelContext { this.configDefinitionRepo = configDefinitionRepo; this.fileRegistry = fileRegistry; this.hostProvisioner = hostProvisioner; + this.provisioned = provisioned; this.properties = properties; this.appDir = appDir; + this.wantedDockerImageRepository = wantedDockerImageRepository; this.modelVespaVersion = modelVespaVersion; this.wantedNodeVespaVersion = wantedNodeVespaVersion; } @@ -97,6 +106,9 @@ public class ModelContextImpl implements ModelContext { public Optional<HostProvisioner> hostProvisioner() { return hostProvisioner; } @Override + public Provisioned provisioned() { return provisioned; } + + @Override public DeployLogger deployLogger() { return deployLogger; } @Override @@ -112,6 +124,9 @@ public class ModelContextImpl implements ModelContext { public Optional<File> appDir() { return appDir; } @Override + public Optional<String> wantedDockerImageRepository() { return wantedDockerImageRepository; } + + @Override public Version modelVespaVersion() { return modelVespaVersion; } @Override @@ -131,8 +146,12 @@ public class ModelContextImpl implements ModelContext { private final boolean isBootstrap; private final boolean isFirstTimeDeployment; private final boolean useAdaptiveDispatch; + private final double defaultTopKprobability; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; + private final double defaultSoftStartSeconds; + private final String proxyProtocol; + private final Optional<AthenzDomain> athenzDomain; public Properties(ApplicationId applicationId, boolean multitenantFromConfig, @@ -146,7 +165,8 @@ public class ModelContextImpl implements ModelContext { boolean isBootstrap, boolean isFirstTimeDeployment, FlagSource flagSource, - Optional<EndpointCertificateSecrets> endpointCertificateSecrets) { + Optional<EndpointCertificateSecrets> endpointCertificateSecrets, + Optional<AthenzDomain> athenzDomain) { this.applicationId = applicationId; this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant"); this.configServerSpecs = configServerSpecs; @@ -163,6 +183,13 @@ public class ModelContextImpl implements ModelContext { this.endpointCertificateSecrets = endpointCertificateSecrets; defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + defaultSoftStartSeconds = Flags.DEFAULT_SOFT_START_SECONDS.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.proxyProtocol = Flags.PROXY_PROTOCOL.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.athenzDomain = athenzDomain; } @Override @@ -214,9 +241,28 @@ public class ModelContextImpl implements ModelContext { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + public double defaultSoftStartSeconds() { + return 0; + } + + @Override + public double defaultTopKProbability() { + return defaultTopKprobability; + } + // TODO: Remove @Override public boolean useBucketSpaceMetric() { return true; } + + @Override + public boolean useNewAthenzFilter() { return true; } + + @Override + public String proxyProtocol() { return proxyProtocol; } + + @Override + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 13ef19f5f5d..09f5178cf6b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -87,17 +87,16 @@ public class ZooKeeperClient { /** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */ private void createZooKeeperNodes() { - if (!configCurator.exists(rootPath.getAbsolute())) { + if ( ! configCurator.exists(rootPath.getAbsolute())) configCurator.createNode(rootPath.getAbsolute()); - } - for (String subPath : Arrays.asList( - ConfigCurator.DEFCONFIGS_ZK_SUBPATH, - ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, - ConfigCurator.USERAPP_ZK_SUBPATH, - ZKApplicationPackage.fileRegistryNode)) { - // TODO The replaceFirst below is hackish. - configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", "")); + for (String subPath : Arrays.asList(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH, + ConfigCurator.USERAPP_ZK_SUBPATH, + ZKApplicationPackage.fileRegistryNode)) { + // TODO: The replaceFirst below is hackish. + configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), + subPath.replaceFirst("/", "")); } } @@ -108,7 +107,6 @@ public class ZooKeeperClient { */ void write(ApplicationPackage app) { logFine("Feeding application config into ZooKeeper"); - // gives lots and lots of debug output: // BasicConfigurator.configure(); try { logFine("Feeding user def files into ZooKeeper"); writeUserDefs(app); @@ -121,43 +119,34 @@ public class ZooKeeperClient { write(app.getMetaData()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + - "Please ensure that cloudconfig_server is started on the config server node(s), " + - "and check the vespa log for configserver errors. ", e); + "Please ensure that cloudconfig_server is started on the config server node(s), " + + "and check the vespa log for configserver errors. ", e); } } private void writeSearchDefinitions(ApplicationPackage app) throws IOException { Collection<NamedReader> sds = app.getSearchDefinitions(); - if (sds.isEmpty()) { - return; - } + if (sds.isEmpty()) return; + + // TODO: Change to SCHEMAS_DIR after March 2020 Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); configCurator.createNode(zkPath.getAbsolute()); - // Ensures that ranking expressions and other files are also fed. + // Ensures that ranking expressions and other files are also written writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); + writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false); for (NamedReader sd : sds) { - String name = sd.getName(); - Reader reader = sd.getReader(); - String data = com.yahoo.io.IOUtils.readAll(reader); - reader.close(); - configCurator.putData(zkPath.getAbsolute(), name, data); + configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader())); + sd.getReader().close(); } } /** * Puts some of the application package files into ZK - see write(app). * - * @param app The application package to use as input. - * @throws java.io.IOException if not able to write to Zookeeper + * @param app the application package to use as input. + * @throws java.io.IOException if not able to write to Zookeeper */ private void writeSomeOf(ApplicationPackage app) throws IOException { - ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() { - @Override - public boolean accept(Path path) { - return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX); - } - }; - // Copy app package files and subdirs into zk // TODO: We should have a way of doing this which doesn't require repeating all the content writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); @@ -169,7 +158,8 @@ public class ZooKeeperClient { getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); writeDir(app.getFile(ApplicationPackage.RULES_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), - srFilter, true); + (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX), + true); writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), xmlFilter, true); @@ -194,20 +184,12 @@ public class ZooKeeperClient { } private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { - writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { - @Override - public boolean accept(Path path) { - return true; - } - }, recurse); + writeDir(file, zooKeeperAppPath, (__) -> true, recurse); } private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { - if (!dir.isDirectory()) { - logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper."); - return; - } - for (ApplicationFile file: listFiles(dir, filenameFilter)) { + if ( ! dir.isDirectory()) return; + for (ApplicationFile file : listFiles(dir, filenameFilter)) { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... if ("CVS".equals(name)) continue; @@ -223,7 +205,8 @@ public class ZooKeeperClient { } /** - * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories. + * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} + * with slightly different semantics: Never filter out directories. */ private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) { List<ApplicationFile> rawList = dir.listFiles(); @@ -243,9 +226,8 @@ public class ZooKeeperClient { } private void writeFile(ApplicationFile file, Path zkPath) throws IOException { - if (!file.exists()) { - return; - } + if ( ! file.exists()) return; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (InputStream inputStream = file.createInputStream()) { inputStream.transferTo(baos); @@ -292,8 +274,8 @@ public class ZooKeeperClient { String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), - vespaVersion.toFullString(), - exportedRegistry); + vespaVersion.toFullString(), + exportedRegistry); } /** @@ -343,9 +325,8 @@ public class ZooKeeperClient { } public void write(AllocatedHosts hosts) throws IOException { - configCurator.putData( - rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), - AllocatedHostsSerializer.toJson(hosts)); + configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), + AllocatedHostsSerializer.toJson(hosts)); } public void write(Map<Version, FileRegistry> fileRegistryMap) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 1418b2a0bcf..2d5cce2d4f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -59,8 +59,7 @@ public class TesterClient { } private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) { - // TODO: Change log level to DEBUG - logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString()); + logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString()); try { return new ProxyResponse(httpClient.execute(request)); } catch (IOException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java index bb752f9b804..0014d66026b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 833c9ca4fba..3c909730902 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -245,7 +245,6 @@ public class ApplicationHandler extends HttpHandler { } private static boolean isTesterStartTestsRequest(HttpRequest request) { - System.out.println(getBindingMatch(request).groupCount()); return getBindingMatch(request).groupCount() == 9 && request.getUri().getPath().contains("/tester/run/"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java index d2fd4efa7dc..e40d9153ad1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java @@ -48,7 +48,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { @Override @SuppressWarnings({"try", "unused"}) public void run() { - try (Lock lock = lock(lockRoot.append(name()))) { + try (Lock lock = curator.lock(lockRoot.append(name()), Duration.ofSeconds(1))) { maintain(); } catch (UncheckedTimeoutException e) { // another config server instance is running this job at the moment; ok @@ -57,12 +57,6 @@ public abstract class Maintainer extends AbstractComponent implements Runnable { } } - private Lock lock(Path path) { - Lock lock = new Lock(path.getAbsolute(), curator); - lock.acquire(Duration.ofSeconds(1)); - return lock; - } - @Override public void deconstruct() { this.service.shutdown(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 34c7dced404..4f64b73b403 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -6,12 +6,13 @@ import com.yahoo.log.LogLevel; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import java.io.IOException; import java.io.InputStream; @@ -42,12 +43,15 @@ public class ClusterMetricsRetriever { private static final List<String> WANTED_METRIC_SERVICES = List.of(VESPA_CONTAINER, VESPA_QRSERVER, VESPA_DISTRIBUTOR); - private static final CloseableHttpClient httpClient = VespaHttpClientBuilder.create() - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(10 * 1000) - .setSocketTimeout(10 * 1000) - .build()) - .build(); + + private static final CloseableHttpClient httpClient = VespaHttpClientBuilder + .create(PoolingHttpClientConnectionManager::new) + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectTimeout(10 * 1000) + .setSocketTimeout(10 * 1000) + .build()) + .build(); /** * Call the metrics API on each host and aggregate the metrics @@ -62,7 +66,7 @@ public class ClusterMetricsRetriever { getHostMetrics(host, clusterMetricsMap) ); - ForkJoinPool threadPool = new ForkJoinPool(5); + ForkJoinPool threadPool = new ForkJoinPool(10); threadPool.submit(retrieveMetricsJob); threadPool.shutdown(); @@ -102,7 +106,7 @@ public class ClusterMetricsRetriever { return slime; } catch (IOException e) { // Usually caused by applications being deleted during metric retrieval - log.warning("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); + log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e)); return new Slime(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index d88fae0a8ef..4fe2ec129d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -35,16 +34,17 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag use4443Upstream; + private final BooleanFlag nginxUpstreamProxyProtocol; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource); + this.nginxUpstreamProxyProtocol = Flags.NGINX_UPSTREAM_PROXY_PROTOCOL.bindTo(flagSource); } @Override public void getConfig(LbServicesConfig.Builder builder) { + builder.nginxUpstreamProxyProtocol(nginxUpstreamProxyProtocol.value()); models.keySet().stream() .sorted() .forEach(tenant -> { @@ -56,10 +56,15 @@ public class LbServicesProducer implements LbServicesConfig.Producer { LbServicesConfig.Tenants.Builder tb = new LbServicesConfig.Tenants.Builder(); apps.stream() .sorted(Comparator.comparing(ApplicationInfo::getApplicationId)) + .filter(applicationInfo -> generateRoutingConfig(applicationInfo.getApplicationId())) .forEach(applicationInfo -> tb.applications(createLbAppIdKey(applicationInfo.getApplicationId()), getAppConfig(applicationInfo))); return tb; } + private boolean generateRoutingConfig(ApplicationId applicationId) { + return ( ! applicationId.instance().isTester()); + } + private String createLbAppIdKey(ApplicationId applicationId) { return applicationId.application().value() + ":" + zone.environment().value() + ":" + zone.region().value() + ":" + applicationId.instance().value(); } @@ -67,8 +72,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); - ab.use4443Upstream( - use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value()); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo))); @@ -92,10 +95,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) { LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder(); hb.hostname(hostInfo.getHostname()); - hostInfo.getServices().stream() - .forEach(serviceInfo -> { - hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo)); - }); + hostInfo.getServices().forEach(serviceInfo -> hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo))); return hb; } @@ -114,12 +114,11 @@ public class LbServicesProducer implements LbServicesConfig.Producer { filter(prop -> !"".equals(prop)).sorted((a, b) -> a.compareTo(b)).collect(Collectors.toList())) .endpointaliases(endpointAliases) .index(Integer.parseInt(serviceInfo.getProperty("index").orElse("999999"))); - serviceInfo.getPorts().stream() - .forEach(portInfo -> { - LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder() - .number(portInfo.getPort()) - .tags(Joiner.on(" ").join(portInfo.getTags())); - sb.ports(pb); + serviceInfo.getPorts().forEach(portInfo -> { + LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder() + .number(portInfo.getPort()) + .tags(Joiner.on(" ").join(portInfo.getTags())); + sb.ports(pb); }); return sb; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index a2fc2bfd6a0..99bec2c9db7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; @@ -83,12 +84,14 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { protected Application buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, + Optional<String> wantedDockerImageRepository, Version wantedNodeVespaVersion, Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models Instant now) { log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", modelFactory.version(), appGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); + Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, Optional.empty(), @@ -96,9 +99,13 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { logger, configDefinitionRepo, getForVersionOrLatest(applicationPackage.getFileRegistries(), modelFactory.version()).orElse(new MockFileRegistry()), - createStaticProvisioner(applicationPackage.getAllocatedHosts(), modelContextProperties), + createStaticProvisioner(applicationPackage.getAllocatedHosts(), + modelContextProperties.applicationId(), + provisioned), + provisioned, modelContextProperties, Optional.empty(), + wantedDockerImageRepository, modelFactory.version(), wantedNodeVespaVersion); MetricUpdater applicationMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(applicationId)); @@ -138,7 +145,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { flagSource, new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) .readEndpointCertificateMetadata(applicationId) - .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets)); + .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets), + zkClient.readAthenzDomain()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 2de64ed145c..cc4378ae05e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; @@ -72,6 +73,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { * and assigns to this SettableOptional such that it can be used after this method returns */ public List<MODELRESULT> buildModels(ApplicationId applicationId, + Optional<String> dockerImageRepository, Version wantedNodeVespaVersion, ApplicationPackage applicationPackage, SettableOptional<AllocatedHosts> allocatedHosts, @@ -104,8 +106,9 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { int majorVersion = majorVersions.get(i); try { allApplicationModels.addAll(buildModelVersions(keepMajorVersion(majorVersion, versions), - applicationId, wantedNodeVespaVersion, applicationPackage, - allocatedHosts, now, buildLatestModelForThisMajor, majorVersion)); + applicationId, dockerImageRepository, wantedNodeVespaVersion, + applicationPackage, allocatedHosts, now, + buildLatestModelForThisMajor, majorVersion)); buildLatestModelForThisMajor = false; // We have successfully built latest model version, do it only for this major } catch (OutOfCapacityException | ApplicationLockException | TransientException e) { @@ -146,6 +149,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { // versions is the set of versions for one particular major version private List<MODELRESULT> buildModelVersions(Set<Version> versions, ApplicationId applicationId, + Optional<String> wantedDockerImageRepository, Version wantedNodeVespaVersion, ApplicationPackage applicationPackage, SettableOptional<AllocatedHosts> allocatedHosts, @@ -160,6 +164,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { MODELRESULT latestModelVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest.get()), applicationPackage, applicationId, + wantedDockerImageRepository, wantedNodeVespaVersion, allocatedHosts.asOptional(), now); @@ -182,6 +187,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { modelVersion = buildModelVersion(modelFactoryRegistry.getFactory(version), applicationPackage, applicationId, + wantedDockerImageRepository, wantedNodeVespaVersion, allocatedHosts.asOptional(), now); @@ -192,8 +198,10 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { // config models (which is always true for manually deployed zones) if (allApplicationVersions.size() > 0 && allApplicationVersions.get(0).getModel().skipOldConfigModels(now)) log.log(LogLevel.INFO, applicationId + ": Skipping old version (due to validation override)"); - else + else { + log.log(LogLevel.ERROR, applicationId + ": Failed to build version " + version); throw e; + } } } return allApplicationVersions; @@ -236,9 +244,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { } protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId, - Version wantedNodeVespaVersion, - Optional<AllocatedHosts> allocatedHosts, + ApplicationId applicationId, Optional<String> dockerImageRepository, + Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts, Instant now); /** @@ -246,15 +253,17 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { * returns empty otherwise, which may either mean that no hosts are allocated or that we are running * non-hosted and should default to use hosts defined in the application package, depending on context */ - Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts, ModelContext.Properties properties) { + Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts, + ApplicationId applicationId, + Provisioned provisioned) { if (hosted && allocatedHosts.isPresent()) - return Optional.of(new StaticProvisioner(allocatedHosts.get(), createNodeRepositoryProvisioner(properties).get())); + return Optional.of(new StaticProvisioner(allocatedHosts.get(), createNodeRepositoryProvisioner(applicationId, provisioned).get())); return Optional.empty(); } - Optional<HostProvisioner> createNodeRepositoryProvisioner(ModelContext.Properties properties) { + Optional<HostProvisioner> createNodeRepositoryProvisioner(ApplicationId applicationId, Provisioned provisioned) { return hostProvisionerProvider.getHostProvisioner().map( - provisioner -> new ProvisionerAdapter(provisioner, properties.applicationId())); + provisioner -> new ProvisionerAdapter(provisioner, applicationId, provisioned)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 9cacb78e53c..6d0403f21c8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -11,6 +11,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; import com.yahoo.config.model.application.provider.FilesApplicationPackage; @@ -82,8 +83,9 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P @Override protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId, - com.yahoo.component.Version wantedNodeVespaVersion, + ApplicationId applicationId, + Optional<String> wantedDockerImageRepository, + Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts, Instant now) { Version modelVersion = modelFactory.version(); @@ -91,6 +93,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir()); // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise + Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, modelOf(modelVersion), @@ -98,9 +101,11 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P logger, configDefinitionRepo, fileDistributionProvider.getFileRegistry(), - createHostProvisioner(allocatedHosts), + createHostProvisioner(allocatedHosts, provisioned), + provisioned, properties, getAppDir(applicationPackage), + wantedDockerImageRepository, modelVersion, wantedNodeVespaVersion); @@ -120,11 +125,15 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P // This method is an excellent demonstration of what happens when one is too liberal with Optional // -bratseth, who had to write the below :-\ - private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts) { - Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties); + private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts, + Provisioned provisioned) { + Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties.applicationId(), + provisioned); if ( ! allocatedHosts.isPresent()) return nodeRepositoryProvisioner; - Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts, properties); + Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts, + properties.applicationId(), + provisioned); if ( ! staticProvisioner.isPresent()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted // Nodes are already allocated by a model and we should use them unless this model requests hosts from a diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java index 87e9fa287a4..249c8639ea9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.monitoring; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -14,14 +16,17 @@ import com.yahoo.statistics.Counter; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** - * Statistics for server. The statistics framework takes care of logging. + * Metrics for config server. The statistics framework takes care of logging. * * @author Harald Musum - * @since 4.2 */ -public class Metrics extends TimerTask implements MetricUpdaterFactory { +public class Metrics extends AbstractComponent implements MetricUpdaterFactory, Runnable { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Metrics.class.getName()); private static final String METRIC_REQUESTS = getMetricName("requests"); @@ -33,23 +38,34 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { private final Counter failedRequests; private final Counter procTimeCounter; private final Metric metric; - private final ZKMetricUpdater zkMetricUpdater; + private final Optional<ZKMetricUpdater> zkMetricUpdater; // TODO The map is the key for now private final Map<Map<String, String>, MetricUpdater> metricUpdaters = new ConcurrentHashMap<>(); - private final Timer timer = new Timer(); + private final Optional<ScheduledExecutorService> executorService; @Inject public Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, ZookeeperServerConfig zkServerConfig) { + this(metric, statistics, healthMonitorConfig, zkServerConfig, true); + } + + private Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, + ZookeeperServerConfig zkServerConfig, boolean createZkMetricUpdater) { this.metric = metric; requests = createCounter(METRIC_REQUESTS, statistics); failedRequests = createCounter(METRIC_FAILED_REQUESTS, statistics); procTimeCounter = createCounter("procTime", statistics); - log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); - long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); - timer.scheduleAtFixedRate(this, 20000, intervalMs); - zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 19500, intervalMs); + if (createZkMetricUpdater) { + log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); + long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); + executorService = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("configserver-metrics"))); + executorService.get().scheduleAtFixedRate(this, 20000, intervalMs, TimeUnit.MILLISECONDS); + zkMetricUpdater = Optional.of(new ZKMetricUpdater(zkServerConfig, 19500, intervalMs)); + } else { + executorService = Optional.empty(); + zkMetricUpdater = Optional.empty(); + } } public static Metrics createTestMetrics() { @@ -58,14 +74,13 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); builder.snapshot_interval(60.0); ZookeeperServerConfig.Builder zkBuilder = new ZookeeperServerConfig.Builder().myid(1); - return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder)); + return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder), false); } private Counter createCounter(String name, Statistics statistics) { return new Counter(name, statistics, false); } - void incrementRequests(Metric.Context metricContext) { requests.increment(1); metric.add(METRIC_REQUESTS, 1, metricContext); @@ -126,8 +141,12 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory { } } setRegularMetrics(); - zkMetricUpdater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null)); - timer.purge(); + zkMetricUpdater.ifPresent(updater -> updater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null))); + } + + public void deconstruct() { + executorService.ifPresent(ExecutorService::shutdown); + zkMetricUpdater.ifPresent(ZKMetricUpdater::shutdown); } private void setRegularMetrics() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java index b2813be5456..62da6fcffbe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.monitoring; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.concurrent.DaemonThreadFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -17,6 +18,8 @@ import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; @@ -26,7 +29,7 @@ import java.util.regex.Pattern; import static com.yahoo.vespa.config.server.monitoring.Metrics.getMetricName; -public class ZKMetricUpdater extends TimerTask { +public class ZKMetricUpdater implements Runnable { private static final Logger log = Logger.getLogger(ZKMetricUpdater.class.getName()); public static final String METRIC_ZK_ZNODES = getMetricName("zkZNodes"); @@ -35,19 +38,19 @@ public class ZKMetricUpdater extends TimerTask { public static final String METRIC_ZK_CONNECTIONS = getMetricName("zkConnections"); public static final String METRIC_ZK_OUTSTANDING_REQUESTS = getMetricName("zkOutstandingRequests"); - private final int CONNECTION_TIMEOUT_MS = 500; - private final int WRITE_TIMEOUT_MS = 250; - private final int READ_TIMEOUT_MS = 500; + private static final int CONNECTION_TIMEOUT_MS = 500; + private static final int WRITE_TIMEOUT_MS = 250; + private static final int READ_TIMEOUT_MS = 500; private AtomicReference<Map<String, Long>> zkMetrics = new AtomicReference<>(new HashMap<>()); - private final Timer timer = new Timer(); + private final ScheduledExecutorService executorService; private final int zkPort; public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delayMS, long intervalMS) { this.zkPort = zkServerConfig.clientPort(); - if (intervalMS > 0) { - timer.scheduleAtFixedRate(this, delayMS, intervalMS); - } + if (intervalMS <= 0 ) throw new IllegalArgumentException("interval must be positive, was " + intervalMS + " ms"); + this.executorService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("zkmetricupdater")); + this.executorService.scheduleAtFixedRate(this, delayMS, intervalMS, TimeUnit.MILLISECONDS); } private void setMetricAttribute(String attribute, long value, Map<String, Long> data) { @@ -74,7 +77,10 @@ public class ZKMetricUpdater extends TimerTask { public void run() { Optional<String> report = retrieveReport(); report.ifPresent(this::parseReport); - timer.purge(); + } + + public void shutdown() { + executorService.shutdown(); } private Optional<String> retrieveReport() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index ee4cc4a3043..7ce255910e6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.provision; import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; @@ -23,10 +24,12 @@ public class ProvisionerAdapter implements HostProvisioner { private final Provisioner provisioner; private final ApplicationId applicationId; + private final Provisioned provisioned; - public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId) { + public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId, Provisioned provisioned) { this.provisioner = provisioner; this.applicationId = applicationId; + this.provisioned = provisioned; } @Override @@ -36,8 +39,18 @@ public class ProvisionerAdapter implements HostProvisioner { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return provisioner.prepare(applicationId, cluster, capacity, groups, logger); + provisioned.add(cluster.id(), capacity); + return provisioner.prepare(applicationId, cluster, capacity.withGroups(groups), logger); } + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + provisioned.add(cluster.id(), capacity); + return provisioner.prepare(applicationId, cluster, capacity, logger); + } + + + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java index 26d322665f0..8ddfb1e8b09 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -33,8 +33,15 @@ public class StaticProvisioner implements HostProvisioner { throw new UnsupportedOperationException("Allocating a single host from provisioning info is not supported"); } + @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { + return prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { List<HostSpec> hostsAlreadyAllocatedToCluster = allocatedHosts.getHosts().stream() .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) @@ -42,7 +49,7 @@ public class StaticProvisioner implements HostProvisioner { if ( ! hostsAlreadyAllocatedToCluster.isEmpty()) return hostsAlreadyAllocatedToCluster; else - return fallback.prepare(cluster, capacity, groups, logger); + return fallback.prepare(cluster, capacity, logger); } private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 67aa49ea0b9..48580dbc6f4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -167,7 +167,7 @@ class GetConfigProcessor implements Runnable { log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); - ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5); + ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5); request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5()); respond(request); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java index dcbc21e536c..5235a2bcadd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java @@ -29,7 +29,7 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory { String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength()); Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes())); - return new SlimeConfigResponse(compressed, defFile, generation, internalRedeploy, configMd5, info); + return new SlimeConfigResponse(compressed, generation, internalRedeploy, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java index 91809f90a70..bd0b117c3db 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java @@ -25,7 +25,7 @@ public class UncompressedConfigResponseFactory implements ConfigResponseFactory Utf8Array rawPayload = payload.toUtf8Array(true); String configMd5 = ConfigUtils.getMd5(rawPayload); CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength()); - return new SlimeConfigResponse(rawPayload, defFile, generation, internalRedeploy, configMd5, info); + return new SlimeConfigResponse(rawPayload, generation, internalRedeploy, configMd5, info); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 2c6d7de8b0c..825ae0d8d92 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.transaction.AbstractTransaction; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -138,6 +139,14 @@ public class LocalSession extends Session implements Comparable<LocalSession> { zooKeeperClient.writeVespaVersion(version); } + public void setDockerImageRepository(Optional<String> dockerImageRepository) { + zooKeeperClient.writeDockerImageRepository(dockerImageRepository); + } + + public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { + zooKeeperClient.writeAthenzDomain(athenzDomain); + } + public enum Mode { READ, WRITE } @@ -148,8 +157,12 @@ public class LocalSession extends Session implements Comparable<LocalSession> { public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); } + public Optional<String> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); } + public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } + public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); } + public AllocatedHosts getAllocatedHosts() { return zooKeeperClient.getAllocatedHosts(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 1a41c1efd7a..f7ed801ddbd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -5,10 +5,11 @@ import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; @@ -35,6 +36,8 @@ public final class PrepareParams { static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints"; static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName"; static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata"; + static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository"; + static final String ATHENZ_DOMAIN = "athenzDomain"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -46,11 +49,14 @@ public final class PrepareParams { private final List<ContainerEndpoint> containerEndpoints; private final Optional<String> tlsSecretsKeyName; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; + private final Optional<String> dockerImageRepository; + private final Optional<AthenzDomain> athenzDomain; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata, + Optional<String> dockerImageRepository, Optional<AthenzDomain> athenzDomain) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; @@ -61,6 +67,8 @@ public final class PrepareParams { this.containerEndpoints = containerEndpoints; this.tlsSecretsKeyName = tlsSecretsKeyName; this.endpointCertificateMetadata = endpointCertificateMetadata; + this.dockerImageRepository = dockerImageRepository; + this.athenzDomain = athenzDomain; } public static class Builder { @@ -75,6 +83,8 @@ public final class PrepareParams { private List<ContainerEndpoint> containerEndpoints = List.of(); private Optional<String> tlsSecretsKeyName = Optional.empty(); private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); + private Optional<String> dockerImageRepository = Optional.empty(); + private Optional<AthenzDomain> athenzDomain = Optional.empty(); public Builder() { } @@ -142,9 +152,26 @@ public final class PrepareParams { return this; } + public Builder dockerImageRepository(String dockerImageRepository) { + if (dockerImageRepository == null) return this; + this.dockerImageRepository = Optional.of(dockerImageRepository); + return this; + } + + public Builder athenzDomain(String athenzDomain) { + this.athenzDomain = Optional.ofNullable(athenzDomain).map(AthenzDomain::from); + return this; + } + + public Builder athenzDomain(AthenzDomain athenzDomain) { + this.athenzDomain = Optional.of(athenzDomain); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata); + verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, + endpointCertificateMetadata, dockerImageRepository, athenzDomain); } } @@ -159,6 +186,8 @@ public final class PrepareParams { .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME)) .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME)) .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME)) + .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY)) + .athenzDomain(request.getProperty(ATHENZ_DOMAIN)) .build(); } @@ -219,4 +248,11 @@ public final class PrepareParams { public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() { return endpointCertificateMetadata; } + + public Optional<String> dockerImageRepository() { + return dockerImageRepository; + } + + public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 26f437920ad..39c15474f0e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -63,6 +63,7 @@ public class RemoteSession extends Session { Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), + zooKeeperClient.readDockerImageRepository(), zooKeeperClient.readVespaVersion(), applicationPackage, new SettableOptional<>(allocatedHosts), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index cb1f6519f57..5ae1289033d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -127,10 +127,13 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { ApplicationId existingApplicationId = existingSession.getApplicationId(); long activeSessionId = getActiveSessionId(existingApplicationId); - logger.log(LogLevel.DEBUG, "Create from existing application id " + existingApplicationId + ", active session id is " + activeSessionId); + logger.log(LogLevel.DEBUG, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); + // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); session.setVespaVersion(existingSession.getVespaVersion()); + session.setDockerImageRepository(existingSession.getDockerImageRepository()); + session.setAthenzDomain(existingSession.getAthenzDomain()); return session; } @@ -138,7 +141,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { boolean internalRedeploy, TimeoutBudget timeoutBudget) { long sessionId = sessionCounter.nextSessionId(); Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Next session id is " + sessionId + " , sessionIdPath=" + sessionIdPath.getAbsolute()); try { ensureZKPathDoesNotExist(sessionIdPath); SessionZooKeeperClient sessionZooKeeperClient = new SessionZooKeeperClient(curator, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 0115876ded9..6c77964a58b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; @@ -139,8 +140,11 @@ public class SessionPreparer { final Path tenantPath; final ApplicationId applicationId; + /** The repository part of docker image to be used for this deployment */ + final Optional<String> dockerImageRepository; + /** The version of Vespa the application to be prepared specifies for its nodes */ - final com.yahoo.component.Version vespaVersion; + final Version vespaVersion; final ContainerEndpointsCache containerEndpoints; final Set<ContainerEndpoint> endpointsSet; @@ -149,6 +153,7 @@ public class SessionPreparer { private final EndpointCertificateRetriever endpointCertificateRetriever; private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata; private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; + private final Optional<AthenzDomain> athenzDomain; private ApplicationPackage applicationPackage; private List<PreparedModelsBuilder.PreparedModelResult> modelResultList; @@ -165,6 +170,7 @@ public class SessionPreparer { this.tenantPath = tenantPath; this.applicationId = params.getApplicationId(); + this.dockerImageRepository = params.dockerImageRepository(); this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator); this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath); @@ -178,6 +184,7 @@ public class SessionPreparer { .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets); this.endpointsSet = getEndpoints(params.containerEndpoints()); + this.athenzDomain = params.athenzDomain(); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), configserverConfig.multitenant(), @@ -191,7 +198,8 @@ public class SessionPreparer { params.isBootstrap(), ! currentActiveApplicationSet.isPresent(), context.getFlagSource(), - endpointCertificateSecrets); + endpointCertificateSecrets, + athenzDomain); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, @@ -223,7 +231,7 @@ public class SessionPreparer { AllocatedHosts buildModels(Instant now) { SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>(); - this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, + this.modelResultList = preparedModelsBuilder.buildModels(applicationId, dockerImageRepository, vespaVersion, applicationPackage, allocatedHosts, now); checkTimeout("build models"); return allocatedHosts.get(); @@ -239,10 +247,12 @@ public class SessionPreparer { writeStateToZooKeeper(context.getSessionZooKeeperClient(), applicationPackage, applicationId, + dockerImageRepository, vespaVersion, logger, prepareResult.getFileRegistries(), - prepareResult.allocatedHosts()); + prepareResult.allocatedHosts(), + athenzDomain); checkTimeout("write state to zookeeper"); } @@ -281,15 +291,20 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, ApplicationId applicationId, - com.yahoo.component.Version vespaVersion, + Optional<String> dockerImageRepository, + Version vespaVersion, DeployLogger deployLogger, Map<Version, FileRegistry> fileRegistryMap, - AllocatedHosts allocatedHosts) { + AllocatedHosts allocatedHosts, + Optional<AthenzDomain> athenzDomain) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); + // Note: When changing the below you need to also change similar calls in SessionFactoryImpl.createSessionFromExisting() zooKeeperClient.writeApplicationId(applicationId); zooKeeperClient.writeVespaVersion(vespaVersion); + zooKeeperClient.writeDockerImageRepository(dockerImageRepository); + zooKeeperClient.writeAthenzDomain(athenzDomain); } catch (RuntimeException | IOException e) { zkDeployer.cleanup(); throw new RuntimeException("Error preparing session", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 727d4e6d7bc..81777bf3642 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -5,23 +5,24 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.transaction.Transaction; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.text.Utf8; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -41,6 +42,8 @@ public class SessionZooKeeperClient { static final String APPLICATION_ID_PATH = "applicationId"; private static final String VERSION_PATH = "version"; private static final String CREATE_TIME_PATH = "createTime"; + private static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository"; + private static final String ATHENZ_DOMAIN = "athenzDomain"; private final Curator curator; private final ConfigCurator configCurator; private final Path sessionPath; @@ -165,6 +168,14 @@ public class SessionZooKeeperClient { return sessionPath.append(VERSION_PATH).getAbsolute(); } + private String dockerImageRepositoryPath() { + return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH).getAbsolute(); + } + + private String athenzDomainPath() { + return sessionPath.append(ATHENZ_DOMAIN).getAbsolute(); + } + public void writeVespaVersion(Version version) { configCurator.putData(versionPath(), version.toString()); } @@ -174,6 +185,16 @@ public class SessionZooKeeperClient { return new Version(configCurator.getData(versionPath())); } + public Optional<String> readDockerImageRepository() { + if ( ! configCurator.exists(dockerImageRepositoryPath())) return Optional.empty(); + String dockerImageRepository = configCurator.getData(dockerImageRepositoryPath()); + return dockerImageRepository.isEmpty() ? Optional.empty() : Optional.of(dockerImageRepository); + } + + public void writeDockerImageRepository(Optional<String> dockerImageRepository) { + dockerImageRepository.ifPresent(repo -> configCurator.putData(dockerImageRepositoryPath(), repo)); + } + // in seconds public long readCreateTime() { String path = getCreateTimePath(); @@ -206,6 +227,17 @@ public class SessionZooKeeperClient { return transaction; } + public void writeAthenzDomain(Optional<AthenzDomain> athenzDomain) { + athenzDomain.ifPresent(domain -> configCurator.putData(athenzDomainPath(), domain.value())); + } + + public Optional<AthenzDomain> readAthenzDomain() { + if ( ! configCurator.exists(athenzDomainPath())) return Optional.empty(); + return Optional.ofNullable(configCurator.getData(athenzDomainPath())) + .filter(domain -> ! domain.isBlank()) + .map(AthenzDomain::from); + } + /** * Create necessary paths atomically for a new session. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java index 65ebb38c2d0..97179e5234b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java index 6500449e557..8e51ac424f9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java index 3ae678969eb..5f180bdaee1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java @@ -9,14 +9,12 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Responsible for providing data from an application subtree in zookeeper. * (i.e. /config/v2/tenants/x/session/<session id for an application>/). * - * Takes care of - * - * * @author Tony Vaagenes */ public class ZKApplication { @@ -84,6 +82,10 @@ public class ZKApplication { return reader(data); } + Optional<Reader> getOptionalDataReader(String path, String node) { + return Optional.ofNullable(getData(path, node)).map(data -> reader(data)); + } + public String getData(String path, String node) { try { return zk.getData(getFullPath(path), node); @@ -181,10 +183,9 @@ public class ZKApplication { } Reader getDataReader(String path) { - final String data = getData(path); - if (data == null) { + String data = getData(path); + if (data == null) throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists"); - } return reader(data); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index bcb958c4b58..c7ec2657996 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -139,13 +139,16 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> searchDefinitionContents() { - List<NamedReader> ret = new ArrayList<>(); - for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) { - if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) { - ret.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd)))); - } + List<NamedReader> schemas = new ArrayList<>(); + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd)))); } - return ret; + for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) { + if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) + schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd)))); + } + return schemas; } @Override @@ -176,7 +179,7 @@ public class ZKApplicationPackage implements ApplicationPackage { try { return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e); + throw new IllegalArgumentException("Could not retrieve config definition " + def, e); } } @@ -264,7 +267,10 @@ public class ZKApplicationPackage implements ApplicationPackage { @Override public Reader getRankingExpression(String name) { - return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name); + Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name); + if (reader.isPresent()) + return reader.get(); + return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name); } @Override diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 970dd49e865..0cd283ca62b 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -63,8 +63,7 @@ <component id="com.yahoo.vespa.service.manager.UnionMonitorManager" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.model.ServiceMonitorImpl" bundle="service-monitor" /> <component id="com.yahoo.vespa.service.duper.DuperModelManager" bundle="service-monitor" /> - <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" /> - <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" /> + <component id="com.yahoo.vespa.orchestrator.status.ZkStatusService" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.controller.RetryingClusterControllerClientFactory" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" /> diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index bec206214f8..bded46dbebe 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -174,6 +174,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ + -Djava.io.tmpdir=${VESPA_HOME}/tmp \ -Djava.library.path=${VESPA_HOME}/lib64 \ -Djava.awt.headless=true \ -Dsun.rmi.dgc.client.gcInterval=3600000 \ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a963252d7ca..0e076d60d52 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -342,11 +342,10 @@ public class ApplicationRepositoryTest { new MockTesterClient(), actual); deployApp(testAppLogServerWithContainer); - Map<String, ?> context = Map.of("tenant", "test1", - "application", "testapp", - "instance", "default", - "environment", "prod", - "region", "default"); + Map<String, ?> context = Map.of("applicationId", "test1.testapp.default", + "tenantName", "test1", + "app", "testapp.default", + "zone", "prod.default"); MockMetric expected = new MockMetric(); expected.set("deployment.prepareMillis", 0L, expected.createContext(context)); expected.set("deployment.activateMillis", 0L, expected.createContext(context)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 2a1254d0d8d..654d811a31f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.config.HealthMonitorConfig; @@ -235,6 +236,7 @@ public class ConfigServerBootstrapTest { private VipStatus createVipStatus(StateMonitor stateMonitor) { return new VipStatus(new QrSearchersConfig.Builder().build(), + new VipStatusConfig.Builder().build(), new ClustersStatus(), stateMonitor); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java index db687857dae..c835bb29ec0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java @@ -15,70 +15,40 @@ public class MockTesterClient extends TesterClient { @Override public HttpResponse getStatus(String testerHostname, int port) { - return new MockStatusResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); + } + }; } @Override public HttpResponse getLog(String testerHostname, int port, Long after) { - return new MockLogResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("log".getBytes(StandardCharsets.UTF_8)); + } + }; } @Override public HttpResponse startTests(String testerHostname, int port, String suite, byte[] config) { - return new MockStartTestsResponse(); + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) { } + }; } @Override public HttpResponse isTesterReady(String testerHostname, int port) { - return new MockTesterReadyResponse(); - } - - private static class MockStatusResponse extends HttpResponse { - - private MockStatusResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("OK".getBytes(StandardCharsets.UTF_8)); - } - - } - - private static class MockLogResponse extends HttpResponse { - - private MockLogResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("log".getBytes(StandardCharsets.UTF_8)); - } - - } - - private static class MockStartTestsResponse extends HttpResponse { - - private MockStartTestsResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) { } - - } - - private static class MockTesterReadyResponse extends HttpResponse { - - private MockTesterReadyResponse() { - super(200); - } - - @Override - public void render(OutputStream outputStream) { } - + return new HttpResponse(200) { + @Override + public void render(OutputStream outputStream) throws IOException { + outputStream.write("{ \"message\": \"OK\" } ".getBytes(StandardCharsets.UTF_8)); + } + }; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 370e0bd3c0e..03c6bad79a8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.component.Version; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.test.MockApplicationPackage; @@ -20,6 +21,7 @@ import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -45,6 +47,7 @@ public class ModelContextImplTest { new StaticConfigDefinitionRepo(), new MockFileRegistry(), Optional.empty(), + new Provisioned(), new ModelContextImpl.Properties( ApplicationId.defaultId(), true, @@ -58,10 +61,12 @@ public class ModelContextImplTest { false, false, flagSource, - null), + null, + Optional.empty()), + Optional.empty(), Optional.empty(), - new Version(6), - new Version(6)); + new Version(7), + new Version(8)); assertTrue(context.applicationPackage() instanceof MockApplicationPackage); assertFalse(context.hostProvisioner().isPresent()); assertFalse(context.permanentApplicationPackage().isPresent()); @@ -75,6 +80,13 @@ public class ModelContextImplTest { assertFalse(context.properties().hostedVespa()); assertThat(context.properties().endpoints(), equalTo(endpoints)); assertThat(context.properties().isFirstTimeDeployment(), equalTo(false)); + + assertEquals(Optional.empty(), context.wantedDockerImageRepository()); + assertEquals(new Version(7), context.modelVespaVersion()); + assertEquals(new Version(8), context.wantedNodeVespaVersion()); + assertEquals(1.0, context.properties().defaultTermwiseLimit(), 0.0); + assertEquals(1.0, context.properties().defaultTopKProbability(), 0.0); + assertFalse(context.properties().useAdaptiveDispatch()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java index f2ee7815df3..9a18570db2d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java @@ -27,7 +27,6 @@ public class ServerCacheTest { private static String configMd5 = "mymd5"; private static String configMd5_2 = "mymd5_2"; private static ConfigDefinition def = new ConfigDefinition("mypayload", new String[0]); - private static ConfigDefinition def_2 = new ConfigDefinition("otherpayload", new String[0]); private static ConfigDefinitionKey fooBarDefKey = new ConfigDefinitionKey("foo", "bar"); private static ConfigDefinitionKey fooBazDefKey = new ConfigDefinitionKey("foo", "baz"); @@ -49,9 +48,9 @@ public class ServerCacheTest { cache = new ServerCache(new TestConfigDefinitionRepo(), userConfigDefinitionRepo); - cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5); - cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5); - cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def_2.getCNode(), 2, false, configMd5_2), configMd5_2); + cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5); + cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5); + cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5_2), configMd5_2); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index b7486dc7951..561422c1cf8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -55,7 +55,7 @@ public class SuperModelControllerTest { ApplicationId app = ApplicationId.from(TenantName.from("a"), ApplicationName.from("foo"), InstanceName.defaultName()); models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp)))); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); } @@ -98,7 +98,7 @@ public class SuperModelControllerTest { models.put(advanced, createApplicationInfo(testApp2, advanced, 4L)); models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); @@ -126,7 +126,7 @@ public class SuperModelControllerTest { models.put(advanced, createApplicationInfo(testApp2, advanced, 4L)); models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L)); - SuperModel superModel = new SuperModel(models); + SuperModel superModel = new SuperModel(models, true); SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory()); LbServicesConfig.Builder lb = new LbServicesConfig.Builder(); han.getSuperModel().getConfig(lb); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java index 405fff3e190..ad910c2afc2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java @@ -4,14 +4,15 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.cloud.config.ModelConfig; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Request; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; @@ -33,12 +34,14 @@ import org.junit.Before; import org.junit.Test; import org.xml.sax.SAXException; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.util.List; +import java.nio.charset.StandardCharsets; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -111,22 +114,27 @@ public class ApplicationTest { } @Test - public void require_that_build_config_can_be_resolved() { - List<String> payload = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)).getLegacyPayload(); - assertTrue(payload.get(1).contains("host")); + public void require_that_build_config_can_be_resolved() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, + ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)) + .serialize(baos, CompressionType.UNCOMPRESSED); + assertTrue(baos.toString().startsWith("{\"vespaVersion\":\"1.0.0\",\"hosts\":[{\"name\":\"mytesthost\"")); } @Test - public void require_that_non_existent_fields_in_schema_is_skipped() { + public void require_that_non_existent_fields_in_schema_is_skipped() throws IOException { // Ask for config without schema and check that we get correct default value back - List<String> payload = handler.resolveConfig(createSimpleConfigRequest()).getLegacyPayload(); - assertThat(payload.get(0), is("boolval false")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + handler.resolveConfig(createSimpleConfigRequest()).serialize(baos, CompressionType.UNCOMPRESSED);; + assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(StandardCharsets.UTF_8)); // Ask for config with wrong schema String[] schema = new String[1]; schema[0] = "boolval bool default=true"; // changed to be true, original is false - payload = handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)).getLegacyPayload(); - assertThat(payload.size(), is(1)); - assertThat(payload.get(0), is("boolval true")); + baos = new ByteArrayOutputStream(); + handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)) + .serialize(baos, CompressionType.UNCOMPRESSED); + assertEquals("{\"boolval\":true,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(Utf8.getCharset())); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 71ae6955e56..1f034a92cbb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import org.junit.Before; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java index beb8abb7be6..88d486cef87 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java @@ -2,15 +2,20 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; +import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; +import java.time.Instant; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -22,7 +27,7 @@ import java.util.function.Function; */ public class OrchestratorMock implements Orchestrator { - private final Set<HostName> suspendedHosts = new HashSet<>(); + private final Map<HostName, HostInfo> hostInfos = new HashMap<>(); private final Set<ApplicationId> suspendedApplications = new HashSet<>(); @Override @@ -32,12 +37,19 @@ public class OrchestratorMock implements Orchestrator { @Override public HostStatus getNodeStatus(HostName hostName) { - return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS; + HostInfo hostInfo = hostInfos.get(hostName); + return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status(); } @Override - public Function<HostName, Optional<HostStatus>> getNodeStatuses() { - return hostName -> Optional.of(getNodeStatus(hostName)); + public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostname) { + HostInfo hostInfo = hostInfos.get(hostname); + return hostInfo == null ? HostInfo.createNoRemarks() : hostInfo; + } + + @Override + public Function<HostName, Optional<HostInfo>> getHostResolver() { + return hostName -> Optional.of(hostInfos.getOrDefault(hostName, HostInfo.createNoRemarks())); } @Override @@ -45,12 +57,12 @@ public class OrchestratorMock implements Orchestrator { @Override public void resume(HostName hostName) { - suspendedHosts.remove(hostName); + hostInfos.remove(hostName); } @Override public void suspend(HostName hostName) { - suspendedHosts.add(hostName); + hostInfos.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH)); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 32b704dd551..84987bce32e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -216,16 +216,32 @@ public class DeployTester { return deployApp(applicationPath, vespaVersion, Instant.now()); } + /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now) { - PrepareParams.Builder paramsBuilder = new PrepareParams.Builder() - .applicationId(applicationId) - .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); + public PrepareResult deployApp(String applicationPath, String vespaVersion, String dockerImageRepository) { + PrepareParams.Builder paramsBuilder = new PrepareParams.Builder(); if (vespaVersion != null) paramsBuilder.vespaVersion(vespaVersion); + return deployApp(applicationPath, Instant.now(), paramsBuilder.dockerImageRepository(dockerImageRepository)); + } + + /** + * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. + */ + public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now) { + return deployApp(applicationPath, now, new PrepareParams.Builder().vespaVersion(vespaVersion)); + } + + /** + * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. + */ + public PrepareResult deployApp(String applicationPath, Instant now, PrepareParams.Builder paramsBuilder) { + paramsBuilder.applicationId(applicationId) + .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); + return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), false, now); } @@ -283,8 +299,14 @@ public class DeployTester { } @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return hostProvisioner.prepare(cluster, capacity, groups, logger); + return hostProvisioner.prepare(cluster, capacity.withGroups(groups), logger); + } + + @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + return hostProvisioner.prepare(cluster, capacity, logger); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index dd0c4eaf342..7e700b78bf7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.session.PrepareParams; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -87,6 +88,24 @@ public class HostedDeployTest { } @Test + public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException { + CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); + DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig()); + String dockerImageRepository = "docker.foo.com:4443/bar/baz"; + tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder() + .vespaVersion("4.5.6") + .dockerImageRepository(dockerImageRepository) + .athenzDomain("foo")); + + Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); + assertTrue(deployment.isPresent()); + deployment.get().activate(); + assertEquals("4.5.6", ((Deployment) deployment.get()).session().getVespaVersion().toString()); + assertEquals(dockerImageRepository, ((Deployment) deployment.get()).session().getDockerImageRepository().get()); + assertEquals("foo", ((Deployment) deployment.get()).session().getAthenzDomain().get().value()); + } + + @Test public void testDeployMultipleVersions() throws IOException { List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")), createHostedModelFactory(Version.fromString("6.2.0")), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java index 9a371639f52..ef0a5f6113d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java @@ -2,34 +2,25 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.protocol.ConfigResponse; - import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import org.junit.Test; import java.io.IOException; -import java.io.StringReader; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; /** * @author Ulf Lilleengen - * @since 5.1 */ public class HttpConfigResponseTest { @Test public void require_that_response_is_created_from_config() throws IOException { final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - InnerCNode targetDef = dParser.getTree(); - ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"); + ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"); HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 3ae98c1b8f2..089b662b797 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -2,28 +2,27 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.config.provision.ApplicationId; - +import com.yahoo.vespa.config.server.rpc.MockRequestHandler; import org.junit.Before; import org.junit.Test; + import java.io.IOException; -import java.io.StringReader; import java.util.Collections; import java.util.HashSet; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static com.yahoo.jdisc.http.HttpResponse.Status.*; +import static com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST; +import static com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -48,8 +47,7 @@ public class HttpGetConfigHandlerTest { // Define config response for mock handler final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } @@ -75,16 +73,10 @@ public class HttpGetConfigHandlerTest { public void require_that_nocache_property_works() throws IOException { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); - mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}")); } - private InnerCNode getInnerCNode() { - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - return dParser.getTree(); - } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 9a326a18dd5..5b0bb7885d8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -55,15 +56,27 @@ public class SessionHandlerTest { public static final String hostname = "foo"; public static final int port = 1337; - public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath, InputStream data) { - return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method, data); + + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, + Cmd cmd, Long id, String subPath, InputStream data, Map<String, String> properties) { + return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + + cmd.toString() + subPath, method, data, properties); + } + + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, + Cmd cmd, Long id, String subPath, InputStream data) { + return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + + cmd.toString() + subPath, method, data); } - public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath) { - return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method); + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, + Cmd cmd, Long id, String subPath) { + return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + + cmd.toString() + subPath, method); } - public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id) { + public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, + Cmd cmd, Long id) { return createTestRequest(path, method, cmd, id, ""); } @@ -88,6 +101,7 @@ public class SessionHandlerTest { private ConfigChangeActions actions = new ConfigChangeActions(); private long createTime = System.currentTimeMillis() / 1000; private ApplicationId applicationId; + private Optional<String> dockerImageRepository; public MockSession(long id, ApplicationPackage app) { this(id, app, new InMemoryFlagSource()); @@ -115,6 +129,7 @@ public class SessionHandlerTest { @Override public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath, Instant now) { status = Session.Status.PREPARE; + this.dockerImageRepository = params.dockerImageRepository(); if (doVerboseLogging) { logger.log(LogLevel.DEBUG, "debuglog"); } @@ -158,6 +173,10 @@ public class SessionHandlerTest { @Override public void delete(NestedTransaction transaction) { } + @Override + public Optional<String> getDockerImageRepository() { + return dockerImageRepository; + } } public enum Cmd { @@ -218,11 +237,17 @@ public class SessionHandlerTest { public Collection<HostSpec> lastHosts; @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { throw new UnsupportedOperationException(); } @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + throw new UnsupportedOperationException(); + } + + @Override public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { activated = true; lastApplicationId = application; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 67677822317..70f66cf8fde 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -28,12 +28,12 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantBuilder; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.After; import org.junit.Before; import org.junit.Test; import javax.ws.rs.client.Client; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets; import java.time.Clock; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; +import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -58,11 +59,16 @@ public class ApplicationHandlerTest { private static File testApp = new File("src/test/apps/app"); - private ListApplicationsHandler listApplicationsHandler; private final static TenantName mytenantName = TenantName.from("mytenant"); private final static TenantName foobar = TenantName.from("foobar"); - private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); - + private final static ApplicationId myTenantApplicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build(); + private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(TenantName.defaultName()).build(); + private final static MockTesterClient testerClient = new MockTesterClient(); + private final static NullMetric metric = new NullMetric(); + private final static ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()); + private static final MockLogRetriever logRetriever = new MockLogRetriever(); + + private TestComponentRegistry componentRegistry; private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; private SessionHandlerTest.MockProvisioner provisioner; @@ -71,27 +77,30 @@ public class ApplicationHandlerTest { @Before public void setup() { - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); + componentRegistry = new TestComponentRegistry.Builder().build(); tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); - tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); provisioner = new SessionHandlerTest.MockProvisioner(); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository(tenantRepository, provisioner, orchestrator, - new ConfigserverConfig(new ConfigserverConfig.Builder()), - new MockLogRetriever(), + configserverConfig, + logRetriever, Clock.systemUTC(), - new MockTesterClient(), - new NullMetric()); - listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), - tenantRepository, - Zone.defaultZone()); + testerClient, + metric); + } + + @After + public void shutdown() { + tenantRepository.close(); } @Test public void testDelete() throws Exception { + tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); + tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); + { applicationRepository.deploy(testApp, prepareParams(applicationId)); Tenant mytenant = tenantRepository.getTenant(applicationId.tenant()); @@ -132,7 +141,7 @@ public class ApplicationHandlerTest { @Test public void testDeleteNonExistent() throws Exception { - deleteAndAssertResponse(applicationId, + deleteAndAssertResponse(myTenantApplicationId, Zone.defaultZone(), Response.Status.NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, @@ -180,10 +189,10 @@ public class ApplicationHandlerTest { InfraDeployerProvider.empty(), new ConfigConvergenceChecker(stateApiFactory), mockHttpProxy, - new ConfigserverConfig(new ConfigserverConfig.Builder()), - new OrchestratorMock(), - new MockTesterClient(), - new NullMetric()); + configserverConfig, + orchestrator, + testerClient, + metric); ApplicationHandler mockHandler = createApplicationHandler(applicationRepository); when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1"))) .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>")); @@ -204,16 +213,15 @@ public class ApplicationHandlerTest { HttpResponse response = fileDistributionStatus(applicationId, zone); assertEquals(200, response.getStatus()); - SessionHandlerTest.getRenderedString(response); assertEquals("{\"hosts\":[{\"hostname\":\"mytesthost\",\"status\":\"UNKNOWN\",\"message\":\"error: Connection error(104)\",\"fileReferences\":[]}],\"status\":\"UNKNOWN\"}", - SessionHandlerTest.getRenderedString(response)); + getRenderedString(response)); // 404 for unknown application - ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant(mytenantName).build(); + ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant("default").build(); HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone); assertEquals(404, responseForUnknown.getStatus()); - assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'mytenant.unknown'\"}", - SessionHandlerTest.getRenderedString(responseForUnknown)); + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'default.unknown'\"}", + getRenderedString(responseForUnknown)); } @Test @@ -225,9 +233,7 @@ public class ApplicationHandlerTest { HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertEquals(200, response.getStatus()); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals("log line", baos.toString()); + assertEquals("log line", getRenderedString(response)); } @Test @@ -235,13 +241,9 @@ public class ApplicationHandlerTest { applicationRepository.deploy(testApp, prepareParams(applicationId)); String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status"; ApplicationHandler mockHandler = createApplicationHandler(); - HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertEquals(200, response.getStatus()); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals("OK", baos.toString()); + assertEquals("OK", getRenderedString(response)); } @Test @@ -252,10 +254,7 @@ public class ApplicationHandlerTest { HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET)); assertEquals(200, response.getStatus()); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - response.render(baos); - assertEquals("log", baos.toString()); + assertEquals("log", getRenderedString(response)); } @Test @@ -345,10 +344,13 @@ public class ApplicationHandlerTest { "/environment/" + zone.environment().value() + "/region/" + zone.region().value() + "/instance/" + applicationId.instance().value() + "\"]"; + ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), + tenantRepository, + Zone.defaultZone()); ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/", - Response.Status.OK, - expected, - com.yahoo.jdisc.http.HttpRequest.Method.GET); + Response.Status.OK, + expected, + com.yahoo.jdisc.http.HttpRequest.Method.GET); } private void restart(ApplicationId application, Zone zone) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index fb09aa99039..b72785876bc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -1,37 +1,36 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; -import static com.yahoo.jdisc.http.HttpRequest.Method.GET; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; -import java.io.IOException; -import java.io.StringReader; -import java.util.Collections; -import java.util.HashSet; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.http.HttpErrorResponse; -import com.yahoo.vespa.config.server.tenant.TenantBuilder; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import org.junit.Before; -import org.junit.Test; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; -import com.yahoo.vespa.config.server.rpc.MockRequestHandler; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpConfigRequest; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.rpc.MockRequestHandler; +import com.yahoo.vespa.config.server.tenant.TenantBuilder; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; + +import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; +import static com.yahoo.jdisc.Response.Status.NOT_FOUND; +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; public class HttpGetConfigHandlerTest { @@ -58,9 +57,8 @@ public class HttpGetConfigHandlerTest { // Define config response for mock handler final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -71,11 +69,10 @@ public class HttpGetConfigHandlerTest { "/application/myapplication/environment/staging/region/myregion/instance/myinstance/foo.bar/myid"; final long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder() .tenant(tenant) .applicationName("myapplication").instanceName("myinstance").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET)); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } @@ -121,18 +118,11 @@ public class HttpGetConfigHandlerTest { public void require_that_nocache_property_works() throws IOException { long generation = 1L; ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder())); - InnerCNode targetDef = getInnerCNode(); mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(), - SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5")); + SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5")); final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true")); HttpResponse response = handler.handle(request); assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING)); } - private InnerCNode getInnerCNode() { - // TODO: Hope to be able to remove this mess soon. - DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n"))); - return dParser.getTree(); - } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 028f5f9eb8c..11cbdb03ccf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -44,6 +44,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; @@ -90,7 +91,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void require_error_when_session_id_does_not_exist() throws Exception { // No session with this id exists - HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 9999L)); + HttpResponse response = request(HttpRequest.Method.PUT, 9999L); assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "Session 9999 was not found"); } @@ -120,8 +121,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_that_activate_url_is_returned_on_success() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertThat(session.getStatus(), is(Session.Status.PREPARE)); assertNotNull(response); assertThat(response.getStatus(), is(OK)); @@ -155,13 +155,11 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_get_response_activate_url_on_ok() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - SessionHandler sessHandler = createHandler(); - sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + request(HttpRequest.Method.PUT, 1L); session.setStatus(Session.Status.PREPARE); SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session); zooKeeperClient.writeStatus(Session.Status.PREPARE); - HttpResponse getResponse = sessHandler.handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L)); + HttpResponse getResponse = request(HttpRequest.Method.GET, 1L); assertResponseContains(getResponse, "\"activate\":\"http://foo:1337" + pathPrefix + "1/active\",\"message\":\"Session 1" + preparedMessage); } @@ -170,19 +168,16 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_get_response_error_on_not_prepared() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - SessionHandler sessHandler = createHandler(); session.setStatus(Session.Status.NEW); SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session); zooKeeperClient.writeStatus(Session.Status.NEW); - HttpResponse getResponse = sessHandler.handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L)); + HttpResponse getResponse = request(HttpRequest.Method.GET, 1L); assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Session not prepared: 1"); session.setStatus(Session.Status.ACTIVATE); zooKeeperClient.writeStatus(Session.Status.ACTIVATE); - getResponse = sessHandler.handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L)); + getResponse = request(HttpRequest.Method.GET, 1L); assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Session is active: 1"); @@ -193,9 +188,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { MockSession session = new MockSession(1, null); localRepo.addSession(session); session.setStatus(Session.Status.ACTIVATE); - SessionHandler sessionHandler = createHandler(); - HttpResponse putResponse = sessionHandler.handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse putResponse = request(HttpRequest.Method.PUT, 1L); assertHttpStatusCodeErrorCodeAndMessage(putResponse, BAD_REQUEST, HttpErrorResponse.errorCodes.BAD_REQUEST, "Session is active: 1"); @@ -205,9 +198,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_get_response_error_when_session_id_does_not_exist() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - SessionHandler sessHandler = createHandler(); - HttpResponse getResponse = sessHandler.handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 9999L)); + HttpResponse getResponse = request(HttpRequest.Method.GET, 9999L); assertHttpStatusCodeErrorCodeAndMessage(getResponse, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "Session 9999 was not found"); @@ -217,8 +208,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_that_tenant_is_in_response() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertNotNull(response); assertThat(response.getStatus(), is(OK)); assertThat(session.getStatus(), is(Session.Status.PREPARE)); @@ -242,8 +232,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { localRepoDefault.addSession(session); pathPrefix = "/application/v2/tenant/" + defaultTenant + "/session/"; - HttpResponse response = handler.handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); + HttpResponse response = request(HttpRequest.Method.PUT, sessionId); assertNotNull(response); assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK)); assertThat(session.getStatus(), is(Session.Status.PREPARE)); @@ -274,8 +263,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_that_config_change_actions_are_in_response() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertResponseContains(response, "\"configChangeActions\":{\"restart\":[],\"refeed\":[]}"); } @@ -289,8 +277,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { new MockRefeedAction("change-id", false, "other change", services, "test"))); MockSession session = new MockSession(1, null, actions); localRepo.addSession(session); - HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertResponseContains(response, "Change(s) between active and new application that require restart:\\nIn cluster 'foo' of type 'bar"); assertResponseContains(response, @@ -301,8 +288,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { public void require_that_config_change_actions_are_not_logged_if_not_existing() throws Exception { MockSession session = new MockSession(1, null); localRepo.addSession(session); - HttpResponse response = createHandler().handle( - SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertResponseNotContains(response, "Change(s) between active and new application that require restart"); assertResponseNotContains(response, "Change(s) between active and new application that require re-feed"); } @@ -312,8 +298,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { String message = "Internal error"; SessionThrowingException session = new SessionThrowingException(new OutOfCapacityException(message)); localRepo.addSession(session); - HttpResponse response = createHandler() - .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertEquals(400, response.getStatus()); Slime data = getData(response); assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.OUT_OF_CAPACITY.name())); @@ -325,8 +310,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { String message = "No nodes available"; SessionThrowingException session = new SessionThrowingException(new NullPointerException(message)); localRepo.addSession(session); - HttpResponse response = createHandler() - .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertEquals(500, response.getStatus()); Slime data = getData(response); assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR.name())); @@ -339,14 +323,22 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { SessionThrowingException session = new SessionThrowingException(new ApplicationLockException(new UncheckedTimeoutException(message))); localRepo.addSession(session); - HttpResponse response = createHandler() - .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L)); + HttpResponse response = request(HttpRequest.Method.PUT, 1L); assertEquals(500, response.getStatus()); Slime data = getData(response); assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.APPLICATION_LOCK_FAILURE.name())); assertThat(data.get().field("message").asString(), is(message)); } + @Test + public void test_docker_image_repository() { + MockSession session = new MockSession(1, null); + localRepo.addSession(session); + String dockerImageRepository = "https://foo.bar.com:4443/baz"; + request(HttpRequest.Method.PUT, 1L, Map.of("dockerImageRepository", dockerImageRepository)); + assertEquals(dockerImageRepository, localRepo.getSession(1).getDockerImageRepository().get()); + } + private Slime getData(HttpResponse response) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); @@ -374,6 +366,14 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { } + private HttpResponse request(HttpRequest.Method put, long l) { + return request(put, l, Map.of()); + } + + private HttpResponse request(HttpRequest.Method put, long l, Map<String, String> requestParameters) { + return createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, put, Cmd.PREPARED, l, "", null, requestParameters)); + } + public static class SessionThrowingException extends LocalSession { private final RuntimeException exception; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java index 0894e38ce09..3f67d8e2cac 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java @@ -14,13 +14,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** @@ -29,11 +31,13 @@ import static org.junit.Assert.*; public class ClusterMetricsRetrieverTest { @Rule - public final WireMockRule wireMock = new WireMockRule(options().port(8080), true); + public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true); @Test public void testMetricAggregation() throws IOException { - List<URI> hosts = List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2"), URI.create("http://localhost:8080/3")); + List<URI> hosts = Stream.of(1, 2, 3) + .map(item -> URI.create("http://localhost:" + wireMock.port() + "/" + item)) + .collect(Collectors.toList()); stubFor(get(urlEqualTo("/1")) .willReturn(aResponse() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 9a7cb72804f..f75d11a145f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; @@ -53,7 +54,7 @@ public class LbServicesProducerTest { private static final Set<ContainerEndpoint> endpoints = Set.of( new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2")) ); - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + private InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final boolean useGlobalServiceId; @Parameterized.Parameters @@ -141,6 +142,23 @@ public class LbServicesProducerTest { assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2)); } + + @Test + public void testRoutingConfigForTesterApplication() throws IOException, SAXException { + assumeFalse(useGlobalServiceId); + + Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder()); + LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel); + LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName); + assertThat(services.servicealiases().size(), is(1)); + assertThat(services.endpointaliases().size(), is(2)); + + // No config for tester application + assertNull(getLbServicesConfig(Zone.defaultZone(), testModel) + .tenants("foo") + .applications("baz:prod:default:custom-t")); + } + private Map<TenantName, Set<ApplicationInfo>> randomizeApplications(Map<TenantName, Set<ApplicationInfo>> testModel, int seed) { Map<TenantName, Set<ApplicationInfo>> randomizedApplications = new LinkedHashMap<>(); List<TenantName> keys = new ArrayList<>(testModel.keySet()); @@ -166,7 +184,7 @@ public class LbServicesProducerTest { Set<ApplicationInfo> aMap = new LinkedHashSet<>(); ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build(); ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build(); - ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").build(); + ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").instanceName("custom-t").build(); // tester app aMap.add(createApplication(fooApp, deploystateBuilder)); aMap.add(createApplication(barApp, deploystateBuilder)); aMap.add(createApplication(bazApp, deploystateBuilder)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java index ed089109759..607a2dca6c6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java @@ -57,12 +57,14 @@ public class ZKMetricUpdaterTest { assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_LATENCY_MAX), equalTo(1234L)); assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_OUTSTANDING_REQUESTS), equalTo(12L)); assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_ZNODES), equalTo(4L)); + + updater.shutdown(); } private ZKMetricUpdater buildUpdater() { ZookeeperServerConfig zkServerConfig = new ZookeeperServerConfig( new ZookeeperServerConfig.Builder().clientPort(serverPort).myid(12345)); - return new ZKMetricUpdater(zkServerConfig, 0, -1); + return new ZKMetricUpdater(zkServerConfig, 0, 100000); } private void setupTcpServer(Supplier<String> reportProvider) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 086dfa5d0d3..0b33de2a42c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -1,18 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; -import com.google.common.base.Joiner; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.component.Version; import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.codegen.DefParser; -import com.yahoo.config.codegen.InnerCNode; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.jrt.Request; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadApplier; @@ -24,12 +21,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3; import com.yahoo.vespa.config.protocol.SlimeConfigResponse; import com.yahoo.vespa.config.protocol.Trace; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.util.ConfigUtils; - import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; @@ -37,11 +33,14 @@ import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; import java.io.IOException; -import java.io.StringReader; import java.util.Optional; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -161,11 +160,7 @@ public class RpcServerTest { builder.intval(123); SimpletypesConfig responseConfig = new SimpletypesConfig(builder); ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig); - InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME, - new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA))) - .getTree(); return SlimeConfigResponse.fromConfigPayload(responsePayload, - targetDef, 3L, true, /* internalRedeploy */ ConfigUtils.getMd5(responsePayload)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 40115170b69..a3b0f3ec44a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -342,11 +342,17 @@ public class SessionPreparerTest { private static class FailWithTransientExceptionProvisioner implements Provisioner { @Override + @Deprecated // TODO: Remove after April 2020 public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); } @Override + public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { + throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); + } + + @Override public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index f2c6aac2bda..9c7da7134e6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.AllocatedHosts; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; @@ -39,7 +40,8 @@ public class ZKApplicationPackageTest { private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty(), - Optional.of(com.yahoo.component.Version.fromString("6.0.1"))))); + Optional.of(Version.fromString("6.0.1")), Optional.empty(), + Optional.empty(), Optional.of(DockerImage.fromString("docker repo"))))); private ConfigCurator configCurator; @@ -59,7 +61,7 @@ public class ZKApplicationPackageTest { assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); - assertEquals(deployState.getSearchDefinitions().size(), 5); + assertEquals(deployState.getSchemas().size(), 5); assertEquals(zkApp.searchDefinitionContents().size(), 5); assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3); @@ -80,6 +82,8 @@ public class ZKApplicationPackageTest { assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS)))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); + // TODO: Enable when dockerImageRepo is written to zk + //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get()); assertTrue(zkApp.getDeployment().isPresent()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); } @@ -87,7 +91,7 @@ public class ZKApplicationPackageTest { private void feed(ConfigCurator zk, File dirToFeed) throws IOException { assertTrue(dirToFeed.isDirectory()); zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); - String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; + String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS)); |