diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2023-03-01 13:37:56 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-01 13:37:56 +0100 |
commit | 7cea54e06359f16427c828b813dee53d6281a42f (patch) | |
tree | f706fa9c185652ec35c5c7d983926e425a64585d | |
parent | 0cd4987fe7c9ff09d49b0020c52efd0cd5aee375 (diff) | |
parent | 3460fa35f934f5bd37436b31396c01d26cff4de0 (diff) |
Merge pull request #26246 from vespa-engine/bratseth/cluster-info
Pass ClusterInfo
32 files changed, 309 insertions, 128 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 5707206019f..f5ad1dce4e7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -106,6 +106,8 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { private final boolean includeSourceFiles; private final TransformerFactory transformerFactory; + private DeploymentSpec deploymentSpec = null; + /** Creates from a directory with source files included */ public static FilesApplicationPackage fromFile(File appDir) { return fromFile(appDir, false); @@ -580,6 +582,12 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { IOUtils.writeFile(metaFile, metaData.asJsonBytes()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + private void preprocessXML(File destination, File inputXml, Zone zone) throws IOException { if ( ! inputXml.exists()) return; try { @@ -589,10 +597,9 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { instance, zone.environment(), zone.region(), - getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) - .map(DeploymentInstanceSpec::tags) - .orElse(Tags.empty())) + getDeploymentSpec().instance(instance) + .map(DeploymentInstanceSpec::tags) + .orElse(Tags.empty())) .run(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java index 93e038c786a..19f1414cd10 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java @@ -110,10 +110,7 @@ public class HostedOverrideProcessorComplexTest { private void assertOverride(InstanceName instance, Environment environment, RegionName region, String expected) throws TransformerException { ApplicationPackage app = FilesApplicationPackage.fromFile(new File(servicesFile).getParentFile()); Document inputDoc = Xml.getDocument(app.getServices()); - Tags tags = app.getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance).map(DeploymentInstanceSpec::tags)) - .orElse(Tags.empty()); + Tags tags = app.getDeploymentSpec().instance(instance).map(DeploymentInstanceSpec::tags).orElse(Tags.empty()); Document newDoc = new OverrideProcessor(instance, environment, region, tags).process(inputDoc); assertEquals(expected, Xml.documentAsString(newDoc, true)); } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 6c83b2029ad..4742d275918 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -84,6 +84,7 @@ public class FilesApplicationPackageTest { assertFalse(new File(appDir, "deployment.xml").exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertFalse(app.getDeployment().isPresent()); + assertTrue(app.getDeploymentSpec().isEmpty()); } @Test @@ -93,6 +94,7 @@ public class FilesApplicationPackageTest { assertTrue(deployment.exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertTrue(app.getDeployment().isPresent()); + assertFalse(app.getDeploymentSpec().isEmpty()); assertFalse(app.getMajorVersion().isPresent()); assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(new FileReader(deployment))); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0ce09c454a0..b6a183c06ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -2,6 +2,7 @@ package com.yahoo.config.application.api; import com.yahoo.component.Version; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -128,20 +129,7 @@ public interface ApplicationPackage { /** Returns the major version this application is valid for, or empty if it is valid for all versions */ default Optional<Integer> getMajorVersion() { - if (getDeployment().isEmpty()) return Optional.empty(); - - Element deployElement = XML.getDocument(getDeployment().get()).getDocumentElement(); - if (deployElement == null) return Optional.empty(); - - String majorVersionString = deployElement.getAttribute("major-version"); - if (majorVersionString == null || majorVersionString.isEmpty()) - return Optional.empty(); - try { - return Optional.of(Integer.parseInt(majorVersionString)); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("major-version must be an integer number, not '" + majorVersionString + "'"); - } + return getDeploymentSpec().majorVersion(); } /** @@ -168,6 +156,19 @@ public interface ApplicationPackage { String getServicesSource(); Optional<Reader> getDeployment(); + + /** + * Returns the parsed deployment spec of this, + * without validating it, and without reparsing on each request. + */ + DeploymentSpec getDeploymentSpec(); + + default DeploymentSpec parseDeploymentSpec(boolean validate) { + return getDeployment() + .map(new DeploymentSpecXmlReader(validate)::read) + .orElse(DeploymentSpec.empty); + } + Optional<Reader> getValidationOverrides(); List<ComponentInfo> getComponentsInfo(Version vespaVersion); @@ -226,6 +227,7 @@ public interface ApplicationPackage { /** * Readers for all the schema files. + * * @return a collection of readers for schemas */ Collection<NamedReader> getSchemas(); @@ -235,10 +237,9 @@ public interface ApplicationPackage { * application package. This is the entry point for the multi environment application package support. This method * will not mutate the existing application package. * - * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove - * @param logger A {@link DeployLogger} to add output that will be returned to the user - * - * @return A new application package instance pointing to a new location + * @param zone a valid {@link Zone} instance, used to decide which parts of services to keep and remove + * @param logger a {@link DeployLogger} to add output that will be returned to the user + * @return a new application package instance pointing to a new location */ default ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException { throw new UnsupportedOperationException("This application package does not support preprocessing"); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java index 8f88b1ba74c..48464904f44 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java @@ -87,27 +87,18 @@ public class Bcp { public static class Group { private final List<RegionMember> members; - private final List<Endpoint> endpoints; + private final Set<RegionName> memberRegions; private final Duration deadline; public Group(List<RegionMember> members, Duration deadline) { - this(members, List.of(), deadline); - } - - public Group(List<RegionMember> members, List<Endpoint> endpoints, Duration deadline) { this.members = List.copyOf(members); - this.endpoints = endpoints; + this.memberRegions = members.stream().map(member -> member.region()).collect(Collectors.toSet()); this.deadline = deadline; } public List<RegionMember> members() { return members; } - /** - * Returns the endpoints defined in this. - * These will be added to instances during XML import post processing - * and should not otherwise be exposed from here. - */ - List<Endpoint> endpoints() { return endpoints; } + public Set<RegionName> memberRegions() { return memberRegions; } /** * Returns the max time until the other regions must be able to handle the additional traffic diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 41644ebc87d..699010417bf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -88,6 +88,8 @@ public class DeploymentSpec { validateBcp(); } + public boolean isEmpty() { return this == empty; } + /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ private void validateTotalDelay(List<Step> steps) { long totalDelaySeconds = steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 91cb2f2622e..d04bb7ecfe0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -519,7 +519,7 @@ public class DeploymentSpecXmlReader { } Duration deadline = XML.attribute("deadline", groupElement).map(value -> toDuration(value, "deadline")).orElse(Duration.ZERO); - groups.add(new Bcp.Group(regions, endpoints, deadline)); + groups.add(new Bcp.Group(regions, deadline)); } validateAndConsolidate(endpointsByZone, zoneEndpoints); return new Bcp(groups); diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index d9918168266..13d87b852e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -7,8 +7,11 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.vespa.model.VespaModel; +import java.time.Duration; +import java.util.Comparator; import java.util.stream.Stream; /** @@ -67,6 +70,18 @@ public final class ConfigModelContext { return ConfigModelContext.create(deployState, vespaModel, configModelRepoAdder, parent, producerId); } + /** Returns a cluster info builder pre-populated with info known in this context. */ + public ClusterInfo.Builder clusterInfo() { + var instance = getApplicationPackage().getDeploymentSpec().instance(properties().applicationId().instance()); + if ( ! instance.isPresent()) return new ClusterInfo.Builder(); + var maxDeadline = instance.get().bcp().groups().stream() + .filter(group -> group.memberRegions().contains(properties().zone().region())) + .map(group -> group.deadline()) + .min(Comparator.comparing(deadline -> deadline)) + .orElse(Duration.ofMinutes(0)); + return new ClusterInfo.Builder().bcpDeadline(maxDeadline); + } + /** * Create an application context from a parent producer and an id. * diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1813e183a60..b8d63ba3778 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -24,6 +24,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; @@ -77,7 +78,7 @@ public class DeployState implements ConfigDefinitionStore { private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; - private final Zone zone; + private final Zone zone; // TODO: Zone is set separately both here and in properties private final QueryProfiles queryProfiles; private final SemanticRules semanticRules; private final ImportedMlModels importedModels; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index dd6087eefc7..41697e61bf2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -129,6 +129,8 @@ public class InMemoryProvisioner implements HostProvisioner { this.retiredHostNames = Set.of(retiredHostNames); } + public Provisioned provisioned() { return provisioned; } + /** May affect e.g. the number of nodes/cluster. */ public InMemoryProvisioner setEnvironment(Environment environment) { this.environment = environment; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index b5999c15fad..3b715c63105 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -57,12 +58,14 @@ public class MockApplicationPackage implements ApplicationPackage { private final List<String> schemas; private final Map<Path, MockApplicationFile> files; private final String schemaDir; - private final Optional<String> deploymentSpec; + private final Optional<String> deploymentSpecString; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; + private DeploymentSpec deploymentSpec = null; + protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, Map<Path, MockApplicationFile> files, String schemaDir, @@ -74,7 +77,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.schemas = schemas; this.files = files; this.schemaDir = schemaDir; - this.deploymentSpec = Optional.ofNullable(deploymentSpec); + this.deploymentSpecString = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), @@ -102,6 +105,12 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + + @Override public Reader getHosts() { if (hostsS == null) return null; return new StringReader(hostsS); @@ -183,7 +192,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Optional<Reader> getDeployment() { - return deploymentSpec.map(StringReader::new); + return deploymentSpecString.map(StringReader::new); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 567ccbfa88b..80000e54b1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -123,7 +123,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterId), context.getDeployLogger(), - false) + false, + context.clusterInfo().build()) .keySet(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index b5fa451fa0b..c968e31325a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.component.Version; @@ -266,8 +267,9 @@ public class NodesSpecification { ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger, - boolean stateful) { - return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful); + boolean stateful, + ClusterInfo clusterInfo) { + return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful, clusterInfo); } public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, @@ -275,7 +277,8 @@ public class NodesSpecification { ClusterSpec.Id clusterId, ZoneEndpoint zoneEndpoint, DeployLogger logger, - boolean stateful) { + boolean stateful, + ClusterInfo info) { if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId) @@ -286,7 +289,7 @@ public class NodesSpecification { .loadBalancerSettings(zoneEndpoint) .stateful(stateful) .build(); - return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount), logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount, info), logger); } private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index a639c158d62..36d34b99223 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; @@ -345,10 +346,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; - Optional<DeploymentSpec> deploymentSpec = app.getDeployment().map(DeploymentSpec::fromXml); + DeploymentSpec deploymentSpec = app.getDeploymentSpec(); if (deploymentSpec.isEmpty()) return; - for (var deprecatedElement : deploymentSpec.get().deprecatedElements()) { + for (var deprecatedElement : deploymentSpec.deprecatedElements()) { deployLogger.logApplicationPackage(WARNING, deprecatedElement.humanReadableString()); } @@ -358,8 +359,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getProperties().ztsUrl(), context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), - deploymentSpec.get()); - addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec.get()); + deploymentSpec); + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec); } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { @@ -863,10 +864,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { InstanceName instance = context.properties().applicationId().instance(); ZoneId zone = ZoneId.from(context.properties().zone().environment(), context.properties().zone().region()); - DeploymentSpec spec = context.getApplicationPackage().getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .orElse(DeploymentSpec.empty); - return spec.zoneEndpoint(instance, zone, cluster); + return context.getApplicationPackage().getDeploymentSpec().zoneEndpoint(instance, zone, cluster); } private static Map<String, String> getEnvironmentVariables(Element environmentVariables) { @@ -924,22 +922,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { // request just enough nodes to satisfy environment capacity requirement - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName())) - .vespaVersion(deployState.getWantedNodeVespaVersion()) - .dockerImageRepository(deployState.getWantedDockerImageRepo()) - .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + - " nodes in " + cluster); - ClusterResources resources = new ClusterResources(nodeCount, 1, NodeResources.unspecified()); - Capacity capacity = Capacity.from(resources, - resources, - IntRange.empty(), - false, - !deployState.getProperties().isBootstrap(), - context.getDeployState().getProperties().cloudAccount()); - var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); + deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + " nodes in " + cluster); + var nodesSpec = NodesSpecification.dedicated(nodeCount, context); + var hosts = nodesSpec.provision(hostSystem, + ClusterSpec.Type.container, + ClusterSpec.Id.from(cluster.getName()), + deployState.getDeployLogger(), + false, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { @@ -956,15 +947,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { - NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); - ClusterSpec.Id clusterId = ClusterSpec.Id.from(cluster.name()); - ZoneEndpoint zoneEndpoint = zoneEndpoint(context, clusterId); + var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, clusterId, - zoneEndpoint, + zoneEndpoint(context, clusterId), log, - getZooKeeper(containerElement) != null); + getZooKeeper(containerElement) != null, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } catch (IllegalArgumentException e) { @@ -998,7 +989,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { StorageGroup.provisionHosts(nodeSpecification, referenceId, cluster.getRoot().hostSystem(), - context.getDeployLogger()); + context); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 31ec764fbde..52b2ce06dfe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -187,10 +187,15 @@ public class StorageGroup { public static Map<HostResource, ClusterMembership> provisionHosts(NodesSpecification nodesSpecification, String clusterIdString, - HostSystem hostSystem, - DeployLogger logger) { + HostSystem hostSystem, + ConfigModelContext context) { ClusterSpec.Id clusterId = ClusterSpec.Id.from(clusterIdString); - return nodesSpecification.provision(hostSystem, ClusterSpec.Type.content, clusterId, logger, true); + return nodesSpecification.provision(hostSystem, + ClusterSpec.Type.content, + clusterId, + context.getDeployLogger(), + true, + context.clusterInfo().build()); } public static class Builder { @@ -203,7 +208,9 @@ public class StorageGroup { this.context = context; } - public StorageGroup buildRootGroup(DeployState deployState, RedundancyBuilder redundancyBuilder, ContentCluster owner) { + public StorageGroup buildRootGroup(DeployState deployState, + RedundancyBuilder redundancyBuilder, + ContentCluster owner) { try { if (owner.isHosted()) validateRedundancyAndGroups(deployState.zone().environment()); @@ -219,7 +226,7 @@ public class StorageGroup { GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null); StorageGroup storageGroup = owner.isHosted() - ? groupBuilder.buildHosted(deployState, owner, Optional.empty()) + ? groupBuilder.buildHosted(deployState, owner, Optional.empty(), context) : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), storageGroup.subgroups.size(), @@ -334,12 +341,18 @@ public class StorageGroup { * @param parent the parent storage group, or empty if this is the root group * @return the storage group build by this */ - public StorageGroup buildHosted(DeployState deployState, ContentCluster owner, Optional<GroupBuilder> parent) { + public StorageGroup buildHosted(DeployState deployState, + ContentCluster owner, + Optional<GroupBuilder> parent, + ConfigModelContext context) { if (storageGroup.getIndex() != null) throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : + provisionHosts(nodeRequirement.get(), + owner.getStorageCluster().getClusterName(), + owner.getRoot().hostSystem(), + context) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -362,7 +375,7 @@ public class StorageGroup { storageGroup.nodes.add(createStorageNode(deployState, owner, host.getKey(), storageGroup, host.getValue())); } for (GroupBuilder subGroup : subGroups) { - storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this))); + storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this), context)); } } return storageGroup; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 137e19e7d86..7f4fc4cd89d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -150,7 +150,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (e != null) setupDocumentProcessing(c, e); } else if (c.persistenceFactory != null) { - throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); + throw new IllegalArgumentException("The <documents> element is mandatory in content cluster '" + clusterId + "'"); } ModelElement tuning = contentElement.child("tuning"); @@ -333,7 +333,8 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger(), - true) + true, + context.clusterInfo().build()) .keySet(); admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState), hosts, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java new file mode 100644 index 00000000000..0abb153696c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -0,0 +1,85 @@ +package com.yahoo.vespa.model; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bratseth + */ +public class ClusterInfoTest { + + @Test + void bcp_deadline_is_passed_in_cluster_info() throws Exception { + var servicesXml = """ + <services version='1.0'> + <container id='testcontainer' version='1.0'> + <nodes count='3'/> + </container> + <content id='testcontent' version='1.0'> + <redundancy>2</redundancy> + <documents/> + </content> + </services> + """; + + var deploymentXml = """ + <deployment version='1.0'> + <prod> + <region>us-west-1</region> + <region>us-east-1</region> + </prod> + <bcp> + <group deadline='30m'> + <region fraction='0.5'>us-east-1</region> + <region>us-west-1</region> + </group> + <group> + <region fraction='0.5'>us-east-1</region> + </group> + </bcp> + </deployment> + """; + + var requestedInUsEast1 = requestedCapacityIn("us-east-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + + var requestedInUsWest1 = requestedCapacityIn("us-west-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + } + + private Map<ClusterSpec.Id, Capacity> requestedCapacityIn(String region, String servicesXml, String deploymentXml) throws Exception { + var applicationPackage = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withDeploymentSpec(deploymentXml) + .build(); + + var provisioner = new InMemoryProvisioner(10, true); + var deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from(region))) + .properties(new TestProperties().setHostedVespa(true) + .setZone(new Zone(Environment.prod, RegionName.from(region)))) + .modelHostProvisioner(provisioner) + .provisioned(provisioner.provisioned()) + .build(); + new VespaModel(new NullConfigModelRegistry(), deployState); + return deployState.provisioned().all(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java index fcc8c82a6e9..a8a063cb5fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; @@ -57,7 +58,8 @@ class CloudAccountChangeValidatorTest { IntRange.empty(), false, false, - Optional.of(cloudAccount).filter(account -> !account.isUnspecified())); + Optional.of(cloudAccount).filter(account -> !account.isUnspecified()), + ClusterInfo.empty()); } private static VespaModel model(Provisioned provisioned) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 6ac4dbd17e3..cad36f5574c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -835,7 +835,7 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); }); - assertTrue(exception.getMessage().contains("The specified content engine requires the <documents> element to be specified.")); + assertEquals("The <documents> element is mandatory in content cluster 'a'", exception.getMessage()); } private ProtonConfig getProtonConfig(ContentCluster content) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 2477d19d46c..c167c862ce4 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -20,6 +20,7 @@ public final class Capacity { private final boolean canFail; private final NodeType type; private final Optional<CloudAccount> cloudAccount; + private final ClusterInfo clusterInfo; private Capacity(ClusterResources min, ClusterResources max, @@ -27,7 +28,8 @@ public final class Capacity { boolean required, boolean canFail, NodeType type, - Optional<CloudAccount> cloudAccount) { + Optional<CloudAccount> cloudAccount, + ClusterInfo clusterInfo) { validate(min); validate(max); if (max.smallerThan(min)) @@ -42,6 +44,7 @@ public final class Capacity { this.canFail = canFail; this.type = type; this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.clusterInfo = clusterInfo; } private static void validate(ClusterResources resources) { @@ -77,12 +80,14 @@ public final class Capacity { return cloudAccount; } + public ClusterInfo clusterInfo() { return clusterInfo; } + public Capacity withLimits(ClusterResources min, ClusterResources max) { return withLimits(min, max, IntRange.empty()); } public Capacity withLimits(ClusterResources min, ClusterResources max, IntRange groupSize) { - return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount); + return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount, clusterInfo); } @Override @@ -98,25 +103,21 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources min, ClusterResources max) { - return from(min, max, IntRange.empty(), false, true, Optional.empty()); + return from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { return from(resources, required, canFail, NodeType.tenant); } - // TODO: Remove after February 2023 - public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty()); - } - - // TODO: Remove after February 2023 - public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount); + // TODO: Remove after March 2023 + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); } - public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount); + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, + Optional<CloudAccount> cloudAccount, ClusterInfo clusterInfo) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, clusterInfo); } /** Creates this from a node type */ @@ -125,7 +126,7 @@ public final class Capacity { } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { - return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty()); + return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty(), ClusterInfo.empty()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java index cec5fb1a1ed..dc01c37f854 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -1,2 +1,40 @@ -package com.yahoo.config.provision;public class ClusterInfo { +package com.yahoo.config.provision; + +import java.time.Duration; + +/** + * Auxiliary information about a cluster, provided by the config model to the node repo during a + * capacity request. + * + * @author bratseth + */ +public class ClusterInfo { + + private static final ClusterInfo empty = new ClusterInfo.Builder().build(); + + private final Duration bcpDeadline; + + private ClusterInfo(Builder builder) { + this.bcpDeadline = builder.bcpDeadline; + } + + public Duration bcpDeadline() { return bcpDeadline; } + + public static ClusterInfo empty() { return empty; } + + public static class Builder { + + private Duration bcpDeadline = Duration.ofMinutes(0); + + public Builder bcpDeadline(Duration duration) { + this.bcpDeadline = duration; + return this; + } + + public ClusterInfo build() { + return new ClusterInfo(this); + } + + } + } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java index 89c0e98b076..a7614bbc016 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java @@ -21,7 +21,8 @@ public class CapacityTest { IntRange.empty(), false, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1, 2, 3, 4)), new ClusterResources(2, 2, new NodeResources(1, 2, 3, 4))); assertValidationFailure(new ClusterResources(4, 4, new NodeResources(1, 2, 3, 4)), @@ -41,7 +42,7 @@ public class CapacityTest { private void assertValidationFailure(ClusterResources min, ClusterResources max) { try { - Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty()); + Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); fail("Expected exception with min " + min + " and max " + max); } catch (IllegalArgumentException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index ffafbb8827e..06c3aa5330e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -270,8 +270,7 @@ public class SessionPreparer { // Validate after doing our own preprocessing on these two files ApplicationMetaData meta = applicationPackage.getMetaData(); InstanceName instance = meta.getApplicationId().instance(); - Tags tags = applicationPackage.getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) + Tags tags = applicationPackage.getDeploymentSpec().instance(instance) .map(DeploymentInstanceSpec::tags) .orElse(Tags.empty()); if (servicesXml.exists()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index fbf14fbdb8c..6d6901136a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; @@ -55,6 +56,8 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { public static final String allocatedHostsNode = "allocatedHosts"; private final ApplicationMetaData metaData; + private DeploymentSpec deploymentSpec = null; + public ZKApplicationPackage(AddFileInterface fileManager, Curator curator, Path sessionPath, int maxNodeSize) { verifyAppPath(curator, sessionPath); zkApplication = new ZKApplication(curator, sessionPath, maxNodeSize); @@ -73,6 +76,12 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { return Optional.of(readAllocatedHosts()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + /** * Reads allocated hosts at the given node. * diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 8f11e171ebe..f3a2c42852f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -101,7 +101,9 @@ public class ZKApplicationPackageTest { assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertEquals(dockerImage, readInfo.getHosts().iterator().next().dockerImageRepo().get().asString()); assertTrue(zkApp.getDeployment().isPresent()); + assertFalse(zkApp.getDeploymentSpec().isEmpty()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); + assertEquals("mydisc", zkApp.getDeploymentSpec().requireInstance("default").globalServiceId().get()); } private void feed(com.yahoo.vespa.curator.Curator zk, File dirToFeed) throws IOException { diff --git a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java index 199ca199667..dc5cf609381 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java @@ -94,11 +94,10 @@ public class BoolFieldValue extends FieldValue { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof BoolFieldValue)) return false; + if ( ! (o instanceof BoolFieldValue other)) return false; if ( ! super.equals(o)) return false; - BoolFieldValue that = (BoolFieldValue) o; - return (value == that.value); + return (value == other.value); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 66d1568262b..0a614cc9b2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ActivationContext; @@ -208,7 +209,8 @@ public class MockNodeRepository extends NodeRepository { IntRange.empty(), false, true, - Optional.empty()), + Optional.empty(), + ClusterInfo.empty()), null), app1Id, provisioner); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 11bc26d90a5..d69d9267cfd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -389,7 +390,7 @@ public class AutoscalingTest { var fixture = DynamicProvisioningTester.fixture() .awsProdSetup(true) .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty())) + .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty(), ClusterInfo.empty())) .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.4, 240); @@ -497,7 +498,7 @@ public class AutoscalingTest { var fixture = DynamicProvisioningTester.fixture() .awsProdSetup(true) .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty())) + .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty(), ClusterInfo.empty())) .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.9, 120); @@ -743,7 +744,7 @@ public class AutoscalingTest { new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); var fixture = DynamicProvisioningTester.fixture() .awsSetup(true, Environment.dev) - .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty())) + .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty(), ClusterInfo.empty())) .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); @@ -763,7 +764,8 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); var fixture = DynamicProvisioningTester.fixture() .hostCount(5) @@ -785,7 +787,8 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); var fixture = DynamicProvisioningTester.fixture() .hostCount(5) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 1e1ccf37c8c..40ca30d758e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -62,10 +63,10 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); // noop diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index 50e19b78905..880a69b61e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -467,7 +468,7 @@ public class HostCapacityMaintainerTest { ClusterSpec spec = ProvisioningTester.contentClusterSpec(); ClusterResources resources = new ClusterResources(2, 1, new NodeResources(16, 24, 100, 1)); CloudAccount cloudAccount0 = CloudAccount.from("000000000000"); - Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); List<HostSpec> prepared = provisioningTester.prepare(applicationId, spec, capacity0); // Hosts are provisioned in requested account @@ -477,7 +478,7 @@ public class HostCapacityMaintainerTest { // Redeployment in different account provisions a new set of hosts CloudAccount cloudAccount1 = CloudAccount.from("100000000000"); - Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); prepared = provisioningTester.prepare(applicationId, spec, capacity1); provisionHostsIn(cloudAccount1, 2, tester); assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 9c69543a9d6..2786da4b69e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; @@ -56,10 +57,10 @@ public class ScalingSuggestionsMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofHours(13)); Duration timeAdded = addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository()); @@ -97,7 +98,7 @@ public class ScalingSuggestionsMaintainerTest { maintainer.maintain(); var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggested().resources().get(); tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofDays(2)); addMeasurements(0.2f, 0.65f, 0.6f, 0, 500, app1, tester.nodeRepository()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index c791c7848d7..d7b5f30a9bc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -6,6 +6,7 @@ import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; @@ -326,7 +327,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_custom_settings() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -343,7 +344,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_changing_visibility() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -379,7 +380,7 @@ public class LoadBalancerProvisionerTest { ClusterResources resources = new ClusterResources(3, 1, nodeResources); CloudAccount cloudAccount0 = CloudAccount.empty; { - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); } LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); @@ -388,7 +389,7 @@ public class LoadBalancerProvisionerTest { // Changing account fails if there is an existing LB in the previous account. CloudAccount cloudAccount1 = CloudAccount.from("111111111111"); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); try { prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"))); fail("Expected exception"); |