diff options
author | Jon Bratseth <bratseth@gmail.com> | 2024-05-19 14:12:41 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-19 14:12:41 +0200 |
commit | cbb09179e61b54c568ed9976194402e935977946 (patch) | |
tree | 464cf634df5a326f7065a6d6a226a1154d8a6fd2 | |
parent | dbd26dbe414187f6f5acbd18785549d9ced2a281 (diff) | |
parent | 12ea6e856952196c38fc47f6b9318900a046f2a0 (diff) |
Merge pull request #31241 from vespa-engine/hakonhall/quota-capacity-policies
Hakonhall/quota capacity policies
40 files changed, 324 insertions, 176 deletions
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index e493840ef82..888a233c62a 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1326,7 +1326,9 @@ "public boolean alwaysMarkPhraseExpensive()", "public boolean sortBlueprintsByCost()", "public int persistenceThreadMaxFeedOpBatchSize()", - "public boolean logserverOtelCol()" + "public boolean logserverOtelCol()", + "public com.yahoo.config.provision.SharedHosts sharedHosts()", + "public com.yahoo.config.provision.NodeResources$Architecture adminClusterArchitecture()" ], "fields" : [ ] }, @@ -1624,6 +1626,9 @@ ], "methods" : [ "public void <init>()", + "public void add(com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity)", + "public java.util.Map clusters()", + "public java.util.Map capacities()", "public void add(com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.Capacity)", "public java.util.Map all()" ], diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index ad5ab5b7ee4..67735329287 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -12,6 +12,9 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DataplaneToken; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.Architecture; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import java.io.File; @@ -114,6 +117,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean sortBlueprintsByCost() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default int persistenceThreadMaxFeedOpBatchSize() { return 1; } @ModelFeatureFlag(owners = {"olaa"}) default boolean logserverOtelCol() { return false; } + @ModelFeatureFlag(owners = {"bratseth"}) default SharedHosts sharedHosts() { return SharedHosts.empty(); } + @ModelFeatureFlag(owners = {"bratseth"}) default Architecture adminClusterArchitecture() { return Architecture.x86_64; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java index db0822c4c10..81e0cb58c7d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java @@ -16,13 +16,28 @@ import java.util.Map; */ public class Provisioned { - private final Map<ClusterSpec.Id, Capacity> provisioned = new HashMap<>(); + private final Map<ClusterSpec.Id, ClusterSpec> clusters = new HashMap<>(); + private final Map<ClusterSpec.Id, Capacity> capacities = new HashMap<>(); + + public void add(ClusterSpec cluster, Capacity capacity) { + clusters.put(cluster.id(), cluster); + capacities.put(cluster.id(), capacity); + } + + /** Returns an unmodifiable map of all the cluster requests recorded during build of the model this belongs to */ + public Map<ClusterSpec.Id, ClusterSpec> clusters() { return Collections.unmodifiableMap(clusters); } + + /** Returns an unmodifiable map of all the capacity provision requests recorded during build of the model this belongs to */ + public Map<ClusterSpec.Id, Capacity> capacities() { return Collections.unmodifiableMap(capacities); } + + // TODO: Remove after June 2024 public void add(ClusterSpec.Id id, Capacity capacity) { - provisioned.put(id, capacity); + capacities.put(id, capacity); } /** Returns an unmodifiable map of all the provision requests recorded during build of the model this belongs to */ - public Map<ClusterSpec.Id, Capacity> all() { return Collections.unmodifiableMap(provisioned); } + // TODO: Remove after June 2024 + public Map<ClusterSpec.Id, Capacity> all() { return Collections.unmodifiableMap(capacities); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index befe57a97e4..da0fd265724 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -158,7 +158,7 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { - provisioned.add(cluster.id(), requested); + provisioned.add(cluster, requested); clusters.add(cluster); if (environment == Environment.dev && ! requested.isRequired()) { requested = requested.withLimits(requested.minResources().withNodes(1), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 4d9386b5f19..ea579aaf5d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -2,12 +2,13 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation.Context; @@ -31,25 +32,35 @@ public class QuotaValidator implements Validator { @Override public void validate(Context context) { + var zone = context.deployState().zone(); + var exclusivity = new Exclusivity(zone, context.deployState().featureFlags().sharedHosts()); + var capacityPolicies = new CapacityPolicies(zone, exclusivity, context.model().applicationPackage().getApplicationId(), + context.deployState().featureFlags().adminClusterArchitecture()); var quota = context.deployState().getProperties().quota(); quota.maxClusterSize().ifPresent(maxClusterSize -> validateMaxClusterSize(maxClusterSize, context.model())); - quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context.model(), context.deployState().getProperties().zone())); + quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context, capacityPolicies)); } - private void validateBudget(BigDecimal budget, VespaModel model, Zone zone) { - var maxSpend = model.allClusters().stream() - .filter(id -> !adminClusterIds(model).contains(id)) - .map(id -> model.provisioned().all().getOrDefault(id, zeroCapacity)) - .mapToDouble(c -> c.maxResources().cost()) // TODO: This may be unspecified -> 0 - .sum(); + private void validateBudget(BigDecimal budget, Context context, + CapacityPolicies capacityPolicies) { + var zone = context.deployState().getProperties().zone(); + var application = context.model().applicationPackage().getApplicationId(); + + var maxSpend = 0.0; + for (var id : context.model().allClusters()) { + if (adminClusterIds(context.model()).contains(id)) continue; + var cluster = context.model().provisioned().clusters().get(id); + var capacity = context.model().provisioned().capacities().getOrDefault(id, zeroCapacity); + maxSpend += capacityPolicies.applyOn(capacity, cluster.isExclusive()).maxResources().cost(); + } - var actualSpend = model.allocatedHosts().getHosts().stream() + var actualSpend = context.model().allocatedHosts().getHosts().stream() .filter(hostSpec -> hostSpec.membership().get().cluster().type() != ClusterSpec.Type.admin) .mapToDouble(hostSpec -> hostSpec.advertisedResources().cost()) .sum(); if (Math.abs(actualSpend) < 0.01) { - log.warning("Deploying application " + model.applicationPackage().getApplicationId() + " with zero budget use. This is suspicious, but not blocked"); + log.warning("Deploying application " + application + " with zero budget use. This is suspicious, but not blocked"); return; } @@ -69,7 +80,7 @@ public class QuotaValidator implements Validator { /** Check that all clusters in the application do not exceed the quota max cluster size. */ private void validateMaxClusterSize(int maxClusterSize, VespaModel model) { - var invalidClusters = model.provisioned().all().entrySet().stream() + var invalidClusters = model.provisioned().capacities().entrySet().stream() .filter(entry -> entry.getValue() != null) .filter(entry -> { var cluster = entry.getValue(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 5d7a8779005..42410dc3acf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -60,9 +60,9 @@ public class ResourcesReductionValidator implements ChangeValidator { * This will always yield specified node resources on hosted instances and never on self-hosted instances. */ private ClusterResources clusterResources(ClusterSpec.Id id, VespaModel model) { - if ( ! model.provisioned().all().containsKey(id)) return null; + if ( ! model.provisioned().capacities().containsKey(id)) return null; - ClusterResources resources = model.provisioned().all().get(id).maxResources(); + ClusterResources resources = model.provisioned().capacities().get(id).maxResources(); if ( ! resources.nodeResources().isUnspecified()) return resources; var containerCluster = model.getContainerClusters().get(id.value()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java index 4df9f261dfe..7aa6eb76995 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -258,7 +258,7 @@ public class ClusterInfoTest { .provisioned(provisioner.provisioned()) .build(); new VespaModel(new NullConfigModelRegistry(), deployState); - return deployState.provisioned().all(); + return deployState.provisioned().capacities(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 89f81dfdaef..590433757c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -54,6 +54,12 @@ public class QuotaValidatorTest { } @Test + void test_deploy_within_quota_budget_because_in_dev() { + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(devZone)); + tester.deploy(null, getServices(10), Environment.dev, null, CONTAINER_CLUSTER); + } + + @Test void test_deploy_above_quota_budget_in_publiccd() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e704da08d18..0e8ce4748b4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -598,7 +598,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setCloudAccount(cloudAccount)) .build()); assertEquals(2, model.hostSystem().getHosts().size()); - assertEquals(List.of(cloudAccount), model.provisioned().all().values() + assertEquals(List.of(cloudAccount), model.provisioned().capacities().values() .stream() .map(capacity -> capacity.cloudAccount().get()) .toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CapacityPolicies.java index 46dfb51a433..818a448187c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CapacityPolicies.java @@ -1,55 +1,44 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; +package com.yahoo.config.provision; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.DiskSpeed; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Map; import java.util.TreeMap; import static com.yahoo.config.provision.NodeResources.Architecture; -import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; import static java.util.Objects.requireNonNull; /** - * Defines the policies for assigning cluster capacity in various environments + * Defines the policies for assigning cluster capacity in various environments. * * @author bratseth - * @see NodeResourceLimits */ public class CapacityPolicies { - private final NodeRepository nodeRepository; private final Zone zone; - private final StringFlag adminClusterNodeArchitecture; + private final Exclusivity exclusivity; + private final ApplicationId applicationId; + private final Architecture adminClusterArchitecture; - public CapacityPolicies(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; - this.zone = nodeRepository.zone(); - this.adminClusterNodeArchitecture = PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(nodeRepository.flagSource()); + public CapacityPolicies(Zone zone, Exclusivity exclusivity, ApplicationId applicationId, Architecture adminClusterArchitecture) { + this.zone = zone; + this.exclusivity = exclusivity; + this.applicationId = applicationId; + this.adminClusterArchitecture = adminClusterArchitecture; } - public Capacity applyOn(Capacity capacity, ApplicationId application, boolean exclusive) { - var min = applyOn(capacity.minResources(), capacity, application, exclusive); - var max = applyOn(capacity.maxResources(), capacity, application, exclusive); + public Capacity applyOn(Capacity capacity, boolean exclusive) { + var min = applyOn(capacity.minResources(), capacity, exclusive); + var max = applyOn(capacity.maxResources(), capacity, exclusive); var groupSize = capacity.groupSize().fromAtMost(max.nodes() / min.groups()) .toAtLeast(min.nodes() / max.groups()); return capacity.withLimits(min, max, groupSize); } - private ClusterResources applyOn(ClusterResources resources, Capacity capacity, ApplicationId application, boolean exclusive) { - int nodes = decideCount(resources.nodes(), capacity.isRequired(), application.instance().isTester()); + private ClusterResources applyOn(ClusterResources resources, Capacity capacity, boolean exclusive) { + int nodes = decideCount(resources.nodes(), capacity.isRequired(), applicationId.instance().isTester()); int groups = decideGroups(resources.nodes(), resources.groups(), nodes); var nodeResources = decideNodeResources(resources.nodeResources(), capacity.isRequired(), exclusive); return new ClusterResources(nodes, groups, nodeResources); @@ -94,31 +83,29 @@ public class CapacityPolicies { return target; } - public ClusterResources specifyFully(ClusterResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.with(specifyFully(resources.nodeResources(), clusterSpec, applicationId)); + public ClusterResources specifyFully(ClusterResources resources, ClusterSpec clusterSpec) { + return resources.with(specifyFully(resources.nodeResources(), clusterSpec)); } - public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec, ApplicationId applicationId) { - return resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec, applicationId).with(DiskSpeed.any)); + public NodeResources specifyFully(NodeResources resources, ClusterSpec clusterSpec) { + return resources.withUnspecifiedFieldsFrom(defaultResources(clusterSpec).with(DiskSpeed.any)); } - private NodeResources defaultResources(ClusterSpec clusterSpec, ApplicationId applicationId) { + private NodeResources defaultResources(ClusterSpec clusterSpec) { if (clusterSpec.type() == ClusterSpec.Type.admin) { - Architecture architecture = adminClusterArchitecture(applicationId); - - if (nodeRepository.exclusiveAllocation(clusterSpec)) { - return smallestExclusiveResources().with(architecture); + if (exclusivity.allocation(clusterSpec)) { + return smallestExclusiveResources().with(adminClusterArchitecture); } if (clusterSpec.id().value().equals("cluster-controllers")) { - return clusterControllerResources(clusterSpec, architecture).with(architecture); + return clusterControllerResources(clusterSpec, adminClusterArchitecture).with(adminClusterArchitecture); } if (clusterSpec.id().value().equals("logserver")) { - return logserverResources(architecture).with(architecture); + return logserverResources(adminClusterArchitecture).with(adminClusterArchitecture); } - return versioned(clusterSpec, Map.of(new Version(0), smallestSharedResources())).with(architecture); + return versioned(clusterSpec, Map.of(new Version(0), smallestSharedResources())).with(adminClusterArchitecture); } if (clusterSpec.type() == ClusterSpec.Type.content) { @@ -157,19 +144,6 @@ public class CapacityPolicies { : new NodeResources(0.5, 2, 50, 0.3); } - private Architecture adminClusterArchitecture(ApplicationId instance) { - return Architecture.valueOf(adminClusterNodeArchitecture.with(INSTANCE_ID, instance.serializedForm()).value()); - } - - /** - * Returns the resources for the newest version not newer than that requested in the cluster spec. - */ - static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { - return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), - "no default resources applicable for " + spec + " among: " + resources) - .getValue(); - } - // The lowest amount of resources that can be exclusive allocated (i.e. a matching host flavor for this exists) private NodeResources smallestExclusiveResources() { return zone.cloud().name() == CloudName.AZURE || zone.cloud().name() == CloudName.GCP @@ -191,4 +165,13 @@ public class CapacityPolicies { return requestedCluster.withExclusivity(exclusive); } + /** + * Returns the resources for the newest version not newer than that requested in the cluster spec. + */ + private static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { + return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), + "no default resources applicable for " + spec + " among: " + resources) + .getValue(); + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Exclusivity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Exclusivity.java new file mode 100644 index 00000000000..6aafb20bee2 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Exclusivity.java @@ -0,0 +1,39 @@ +package com.yahoo.config.provision; + +/** + * A class which can be asked if allocations should be exclusive. + * + * @author bratseth + */ +public class Exclusivity { + + private final Zone zone; + private final SharedHosts sharedHost; + + public Exclusivity(Zone zone, SharedHosts sharedHost) { + this.zone = zone; + this.sharedHost = sharedHost; + } + + /** Returns whether nodes must be allocated to hosts that are exclusive to the cluster type. */ + public boolean clusterType(ClusterSpec cluster) { + return sharedHost.hasClusterType(cluster.type()); + } + + /** Returns whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application. */ + public boolean provisioning(ClusterSpec clusterSpec) { + return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); + } + + /** + * Returns whether nodes are allocated exclusively in this instance given this cluster spec. + * Exclusive allocation requires that the wanted node resources matches the advertised resources of the node + * perfectly. + */ + public boolean allocation(ClusterSpec clusterSpec) { + return clusterSpec.isExclusive() || + ( clusterSpec.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || + ( !zone.cloud().allowHostSharing() && !sharedHost.supportsClusterType(clusterSpec.type())); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SharedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SharedHosts.java new file mode 100644 index 00000000000..ec2660f45c8 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SharedHosts.java @@ -0,0 +1,23 @@ +package com.yahoo.config.provision; + +/** + * @author hakonhall + */ +public interface SharedHosts { + + /** Whether there are any shared hosts specifically for the given cluster type, or without a cluster type restriction. */ + boolean supportsClusterType(ClusterSpec.Type clusterType); + + /** Whether there are any shared hosts specifically for the given cluster type. */ + boolean hasClusterType(ClusterSpec.Type clusterType); + + static SharedHosts empty() { return ofConstant(false, false); } + + static SharedHosts ofConstant(boolean supportsClusterType, boolean hasClusterType) { + return new SharedHosts() { + @Override public boolean supportsClusterType(ClusterSpec.Type clusterType) { return supportsClusterType; } + @Override public boolean hasClusterType(ClusterSpec.Type clusterType) { return hasClusterType; } + }; + } + +} diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml index 5190a65770e..1bb49f3b422 100644 --- a/configserver-flags/pom.xml +++ b/configserver-flags/pom.xml @@ -44,6 +44,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>annotations</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java index f30d329cf09..7a0b3695e7c 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/ConfigServerFlagSource.java @@ -13,6 +13,7 @@ import java.nio.file.FileSystems; * @author hakonhall */ public class ConfigServerFlagSource extends OrderedFlagSource { + @Inject public ConfigServerFlagSource(FlagsDb flagsDb) { this(FileSystems.getDefault(), flagsDb); @@ -21,4 +22,5 @@ public class ConfigServerFlagSource extends OrderedFlagSource { ConfigServerFlagSource(FileSystem fileSystem, FlagsDb flagsDb) { super(new BootstrapFlagSource(fileSystem), new ZooKeeperFlagSource(flagsDb)); } + } diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java index f68c4657d8a..e57088416a0 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/db/ZooKeeperFlagSource.java @@ -13,6 +13,7 @@ import java.util.Optional; * @author hakonhall */ public class ZooKeeperFlagSource implements FlagSource { + private final FlagsDb flagsDb; public ZooKeeperFlagSource(FlagsDb flagsDb) { @@ -23,4 +24,5 @@ public class ZooKeeperFlagSource implements FlagSource { public Optional<RawFlag> fetch(FlagId id, FetchVector vector) { return flagsDb.getValue(id).flatMap(data -> data.resolve(vector)); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 07a8179732f..22199dfc5b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -1136,7 +1136,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public double getQuotaUsageRate(ApplicationId applicationId) { var application = getApplication(applicationId); - return application.getModel().provisioned().all().values().stream() + return application.getModel().provisioned().capacities().values().stream() .map(Capacity::maxResources)// TODO: This may be unspecified -> 0 .mapToDouble(resources -> resources.nodes() * resources.nodeResources().cost()) .sum(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7189ae12c66..c8d3574182a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -25,6 +25,9 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DataplaneToken; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeResources.Architecture; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever; @@ -208,6 +211,8 @@ public class ModelContextImpl implements ModelContext { private final int searchHandlerThreadpool; private final int persistenceThreadMaxFeedOpBatchSize; private final boolean logserverOtelCol; + private final SharedHosts sharedHosts; + private final Architecture adminClusterArchitecture; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -252,6 +257,8 @@ public class ModelContextImpl implements ModelContext { this.sortBlueprintsByCost = flagValue(source, appId, version, Flags.SORT_BLUEPRINTS_BY_COST); this.persistenceThreadMaxFeedOpBatchSize = flagValue(source, appId, version, Flags.PERSISTENCE_THREAD_MAX_FEED_OP_BATCH_SIZE); this.logserverOtelCol = flagValue(source, appId, version, Flags.LOGSERVER_OTELCOL_AGENT); + this.sharedHosts = flagValue(source, appId, version, PermanentFlags.SHARED_HOST); + this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -304,6 +311,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean sortBlueprintsByCost() { return sortBlueprintsByCost; } @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } @Override public boolean logserverOtelCol() { return logserverOtelCol; } + @Override public SharedHosts sharedHosts() { return sharedHosts; } + @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index 956573ecbd4..27805b64e79 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -44,7 +44,7 @@ public class ProvisionerAdapter implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { - provisioned.add(cluster.id(), capacity); + provisioned.add(cluster, capacity); return provisioner.prepare(applicationId, cluster, capacity, logger); } diff --git a/flags/pom.xml b/flags/pom.xml index 9f3a9d35831..38e946a62e8 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -44,6 +44,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <scope>provided</scope> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java index 66356d979a4..3f229862d7a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/custom/SharedHost.java @@ -7,6 +7,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.vespa.flags.PermanentFlags; import java.util.List; @@ -19,7 +21,7 @@ import java.util.Objects; */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(value = JsonInclude.Include.NON_NULL) -public class SharedHost { +public class SharedHost implements SharedHosts { private final List<HostResources> resources; @@ -43,14 +45,16 @@ public class SharedHost { /** Whether there are any shared hosts specifically for the given cluster type, or without a cluster type restriction. */ @JsonIgnore - public boolean supportsClusterType(String clusterType) { - return resources.stream().anyMatch(resource -> resource.clusterType().map(clusterType::equalsIgnoreCase).orElse(true)); + @Override + public boolean supportsClusterType(ClusterSpec.Type clusterType) { + return resources.stream().anyMatch(resource -> resource.clusterType().map(type -> clusterType.name().equalsIgnoreCase(type)).orElse(true)); } /** Whether there are any shared hosts specifically for the given cluster type. */ @JsonIgnore - public boolean hasClusterType(String clusterType) { - return resources.stream().anyMatch(resource -> resource.clusterType().map(clusterType::equalsIgnoreCase).orElse(false)); + @Override + public boolean hasClusterType(ClusterSpec.Type clusterType) { + return resources.stream().anyMatch(resource -> resource.clusterType().map(type -> clusterType.name().equalsIgnoreCase(type)).orElse(false)); } @JsonIgnore diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 539f3128091..cc46fc381f0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -4,25 +4,25 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.maintenance.JobControl; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; -import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources.Architecture; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.JacksonFlag; import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -45,6 +45,8 @@ import java.time.Clock; import java.util.List; import java.util.Optional; +import static com.yahoo.vespa.flags.Dimension.INSTANCE_ID; + /** * The top level singleton in the node repo, providing access to all its state as child objects. * @@ -55,6 +57,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr private final CuratorDb db; private final Clock clock; private final Zone zone; + private final Exclusivity exclusivity; private final Nodes nodes; private final NodeFlavors flavors; private final HostResourcesCalculator resourcesCalculator; @@ -73,7 +76,6 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr private final Orchestrator orchestrator; private final int spareCount; private final ProtoHealthChecker healthChecker; - private final JacksonFlag<SharedHost> sharedHosts; /** * Creates a node repository from a zookeeper provider. @@ -85,6 +87,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr ProvisionServiceProvider provisionServiceProvider, Curator curator, Zone zone, + Exclusivity exclusivity, FlagSource flagSource, MetricsDb metricsDb, Orchestrator orchestrator) { @@ -93,6 +96,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr curator, Clock.systemUTC(), zone, + exclusivity, new DnsNameResolver(), DockerImage.fromString(config.containerImage()), optionalImage(config.tenantContainerImage()), @@ -113,6 +117,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr Curator curator, Clock clock, Zone zone, + Exclusivity exclusivity, NameResolver nameResolver, DockerImage containerImage, Optional<DockerImage> tenantContainerImage, @@ -129,8 +134,9 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr this.flagSource = flagSource; this.db = new CuratorDb(flavors, curator, clock, useCuratorClientCache); - this.zone = zone; this.clock = clock; + this.zone = zone; + this.exclusivity = exclusivity; this.applications = new Applications(db); this.nodes = new Nodes(db, zone, clock, orchestrator, applications); this.flavors = flavors; @@ -147,7 +153,6 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr this.metricsDb = metricsDb; this.orchestrator = orchestrator; this.spareCount = spareCount; - this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(flagSource()); this.healthChecker = provisionServiceProvider.getHealthChecker(); nodes.rewrite(); } @@ -208,25 +213,14 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr /** The number of nodes we should ensure has free capacity for node failures whenever possible */ public int spareCount() { return spareCount; } - /** Returns whether nodes must be allocated to hosts that are exclusive to the cluster type. */ - public boolean exclusiveClusterType(ClusterSpec cluster) { - return sharedHosts.value().hasClusterType(cluster.type().name()); - } - - /** - * Returns whether nodes are allocated exclusively in this instance given this cluster spec. - * Exclusive allocation requires that the wanted node resources matches the advertised resources of the node - * perfectly. - */ - public boolean exclusiveAllocation(ClusterSpec clusterSpec) { - return clusterSpec.isExclusive() || - ( clusterSpec.type().isContainer() && zone.system().isPublic() && !zone.environment().isTest() ) || - ( !zone().cloud().allowHostSharing() && !sharedHosts.value().supportsClusterType(clusterSpec.type().name())); - } + public Exclusivity exclusivity() { return exclusivity; } - /** Whether the nodes of this cluster must be running on hosts that are specifically provisioned for the application. */ - public boolean exclusiveProvisioning(ClusterSpec clusterSpec) { - return !zone.cloud().allowHostSharing() && clusterSpec.isExclusive(); + public CapacityPolicies capacityPoliciesFor(ApplicationId applicationId) { + String adminClusterNodeArchitecture = PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE + .bindTo(flagSource) + .with(INSTANCE_ID, applicationId.serializedForm()) + .value(); + return new CapacityPolicies(zone, exclusivity, applicationId, Architecture.valueOf(adminClusterNodeArchitecture)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index 75a00fa951e..98c5af2688a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -41,7 +41,7 @@ public class AllocatableResources { this.nodes = requested.nodes(); this.groups = requested.groups(); this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), cloudAccount, - nodeRepository.exclusiveAllocation(clusterSpec), false); + nodeRepository.exclusivity().allocation(clusterSpec), false); this.advertisedResources = requested.nodeResources(); this.clusterSpec = clusterSpec; this.fulfilment = 1; @@ -178,7 +178,7 @@ public class AllocatableResources { ClusterModel model, NodeRepository nodeRepository) { var systemLimits = nodeRepository.nodeResourceLimits(); - boolean exclusive = nodeRepository.exclusiveAllocation(clusterSpec); + boolean exclusive = nodeRepository.exclusivity().allocation(clusterSpec); if (! exclusive) { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources var allocatableResources = calculateAllocatableResources(wantedResources, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 5a790a1fe19..82199888a48 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -103,7 +103,7 @@ public class AllocationOptimizer { /** Returns the max resources of a host one node may allocate. */ private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel model) { - if (nodeRepository.exclusiveAllocation(model.clusterSpec())) return hostResources; + if (nodeRepository.exclusivity().allocation(model.clusterSpec())) return hostResources; // static, shared hosts: Allocate at most half of the host cpu to simplify management return hostResources.withVcpu(hostResources.vcpu() / 2); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 74907feee41..504965f1992 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; +import com.yahoo.config.provision.CapacityPolicies; import java.time.Clock; import java.time.Duration; @@ -48,6 +48,7 @@ public class ClusterModel { private static final double fixedCpuCostFraction = 0.1; private final NodeRepository nodeRepository; + private final CapacityPolicies capacityPolicies; private final Application application; private final ClusterSpec clusterSpec; private final Cluster cluster; @@ -84,6 +85,7 @@ public class ClusterModel { MetricsDb metricsDb, Clock clock) { this.nodeRepository = nodeRepository; + this.capacityPolicies = nodeRepository.capacityPoliciesFor(application.id()); this.application = application; this.clusterSpec = clusterSpec; this.cluster = cluster; @@ -108,6 +110,7 @@ public class ClusterModel { ClusterTimeseries clusterTimeseries, ClusterNodesTimeseries nodeTimeseries) { this.nodeRepository = nodeRepository; + this.capacityPolicies = nodeRepository.capacityPoliciesFor(application.id()); this.application = application; this.clusterSpec = clusterSpec; this.cluster = cluster; @@ -171,7 +174,7 @@ public class ClusterModel { } public boolean isExclusive() { - return nodeRepository.exclusiveAllocation(clusterSpec); + return nodeRepository.exclusivity().allocation(clusterSpec); } /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ @@ -436,12 +439,10 @@ public class ClusterModel { double averageReal() { if (nodes.isEmpty()) { // we're estimating - var initialResources = new CapacityPolicies(nodeRepository).specifyFully(cluster.minResources().nodeResources(), - clusterSpec, - application.id()); + var initialResources = capacityPolicies.specifyFully(cluster.minResources().nodeResources(), clusterSpec); return nodeRepository.resourcesCalculator().requestToReal(initialResources, cloudAccount(), - nodeRepository.exclusiveAllocation(clusterSpec), + nodeRepository.exclusivity().allocation(clusterSpec), false).memoryGb(); } else { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index ab93e585c88..51046df90af 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import java.util.Objects; @@ -68,9 +67,9 @@ public class Limits { public Limits fullySpecified(ClusterSpec clusterSpec, NodeRepository nodeRepository, ApplicationId applicationId) { if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); - var capacityPolicies = new CapacityPolicies(nodeRepository); - return new Limits(capacityPolicies.specifyFully(min, clusterSpec, applicationId), - capacityPolicies.specifyFully(max, clusterSpec, applicationId), + var capacityPolicies = nodeRepository.capacityPoliciesFor(applicationId); + return new Limits(capacityPolicies.specifyFully(min, clusterSpec), + capacityPolicies.specifyFully(max, clusterSpec), groupSize); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 98f0dfebc4a..c9e920dc243 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -223,7 +223,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); - HostSharing sharingMode = nodeRepository().exclusiveAllocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; + HostSharing sharingMode = nodeRepository().exclusivity().allocation(asSpec(clusterType, 0)) ? HostSharing.exclusive : HostSharing.shared; HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, sharingMode, clusterType.map(ClusterSpec.Type::valueOf), Optional.empty(), @@ -290,13 +290,13 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { NodePrioritizer prioritizer = new NodePrioritizer(allNodes, application, cluster, nodeSpec, true, false, allocationContext, nodeRepository().nodes(), nodeRepository().resourcesCalculator(), nodeRepository().spareCount(), - nodeRepository().exclusiveAllocation(cluster)); + nodeRepository().exclusivity().allocation(cluster)); List<NodeCandidate> nodeCandidates = prioritizer.collect() .stream() .filter(node -> node.violatesExclusivity(cluster, application, - nodeRepository().exclusiveClusterType(cluster), - nodeRepository().exclusiveAllocation(cluster), + nodeRepository().exclusivity().clusterType(cluster), + nodeRepository().exclusivity().allocation(cluster), false, nodeRepository().zone().cloud().allowHostSharing(), allNodes) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index b149a9af2c2..433bd0bbefe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -206,9 +206,9 @@ class NodeAllocation { private NodeCandidate.ExclusivityViolation violatesExclusivity(NodeCandidate candidate) { return candidate.violatesExclusivity(cluster, application, - nodeRepository.exclusiveClusterType(cluster), - nodeRepository.exclusiveAllocation(cluster), - nodeRepository.exclusiveProvisioning(cluster), + nodeRepository.exclusivity().clusterType(cluster), + nodeRepository.exclusivity().allocation(cluster), + nodeRepository.exclusivity().provisioning(cluster), nodeRepository.zone().cloud().allowHostSharing(), allNodes); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 2588b02d712..8c056ad1a93 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -53,7 +54,6 @@ public class NodeRepositoryProvisioner implements Provisioner { private final NodeRepository nodeRepository; private final AllocationOptimizer allocationOptimizer; - private final CapacityPolicies capacityPolicies; private final Zone zone; private final Preparer preparer; private final Activator activator; @@ -66,7 +66,6 @@ public class NodeRepositoryProvisioner implements Provisioner { Metric metric) { this.nodeRepository = nodeRepository; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); - this.capacityPolicies = new CapacityPolicies(nodeRepository); this.zone = zone; this.loadBalancerProvisioner = provisionServiceProvider.getLoadBalancerService() .map(lbService -> new LoadBalancerProvisioner(nodeRepository, lbService)); @@ -88,23 +87,24 @@ public class NodeRepositoryProvisioner implements Provisioner { " for application " + application + ", cluster " + cluster); validate(application, cluster, requested, logger); + var capacityPolicies = nodeRepository.capacityPoliciesFor(application); NodeResources resources; NodeSpec nodeSpec; if (requested.type() == NodeType.tenant) { cluster = capacityPolicies.decideExclusivity(requested, cluster); - Capacity actual = capacityPolicies.applyOn(requested, application, cluster.isExclusive()); - ClusterResources target = decideTargetResources(application, cluster, actual); + Capacity actual = capacityPolicies.applyOn(requested, cluster.isExclusive()); + ClusterResources target = decideTargetResources(application, cluster, actual, capacityPolicies); validate(actual, target, cluster, application); logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); - resources = getNodeResources(cluster, target.nodeResources(), application); + resources = getNodeResources(cluster, target.nodeResources(), application, capacityPolicies); nodeSpec = NodeSpec.from(target.nodes(), target.groups(), resources, cluster.isExclusive(), actual.canFail(), requested.cloudAccount().orElse(nodeRepository.zone().cloud().account()), requested.clusterInfo().hostTTL()); } else { cluster = cluster.withExclusivity(true); - resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); + resources = getNodeResources(cluster, requested.minResources().nodeResources(), application, capacityPolicies); nodeSpec = NodeSpec.from(requested.type(), nodeRepository.zone().cloud().account()); } return asSortedHosts(preparer.prepare(application, cluster, nodeSpec), @@ -133,8 +133,8 @@ public class NodeRepositoryProvisioner implements Provisioner { } } - private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { - return capacityPolicies.specifyFully(nodeResources, cluster, applicationId); + private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId, CapacityPolicies capacityPolicies) { + return capacityPolicies.specifyFully(nodeResources, cluster); } @Override @@ -166,13 +166,14 @@ public class NodeRepositoryProvisioner implements Provisioner { * Returns the target cluster resources, a value between the min and max in the requested capacity, * and updates the application store with the received min and max. */ - private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested) { + private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity requested, + CapacityPolicies capacityPolicies) { try (Mutex lock = nodeRepository.applications().lock(applicationId)) { var application = nodeRepository.applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(clusterSpec.id(), clusterSpec.isExclusive(), requested); nodeRepository.applications().put(application, lock); var cluster = application.cluster(clusterSpec.id()).get(); - return cluster.target().resources().orElseGet(() -> currentResources(application, clusterSpec, cluster, requested)); + return cluster.target().resources().orElseGet(() -> currentResources(application, clusterSpec, cluster, requested, capacityPolicies)); } } @@ -180,7 +181,8 @@ public class NodeRepositoryProvisioner implements Provisioner { private ClusterResources currentResources(Application application, ClusterSpec clusterSpec, Cluster cluster, - Capacity requested) { + Capacity requested, + CapacityPolicies capacityPolicies) { NodeList nodes = nodeRepository.nodes().list(Node.State.active).owner(application.id()) .cluster(clusterSpec.id()) .not().retired() @@ -188,15 +190,15 @@ public class NodeRepositoryProvisioner implements Provisioner { boolean firstDeployment = nodes.isEmpty(); var current = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, + ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id(), capacityPolicies), clusterSpec, nodeRepository, requested.cloudAccount().orElse(CloudAccount.empty)) : new AllocatableResources(nodes, nodeRepository); var model = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); return within(Limits.of(requested), model, firstDeployment); } - private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { - return capacityPolicies.specifyFully(requested.minResources(), clusterSpec, applicationId); + private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId, CapacityPolicies capacityPolicies) { + return capacityPolicies.specifyFully(requested.minResources(), clusterSpec); } @@ -278,7 +280,7 @@ public class NodeRepositoryProvisioner implements Provisioner { private IllegalArgumentException newNoAllocationPossible(ClusterSpec spec, Limits limits) { StringBuilder message = new StringBuilder("No allocation possible within ").append(limits); - if (nodeRepository.exclusiveAllocation(spec) && findNearestNodeResources(limits).isPresent()) + if (nodeRepository.exclusivity().allocation(spec) && findNearestNodeResources(limits).isPresent()) message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits).get()); return new IllegalArgumentException(message.toString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index 194207ca69d..5eb8c2e7fd7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; @@ -27,7 +28,7 @@ public class NodeResourceLimits { /** Validates the resources applications ask for (which are in "advertised" resource space) */ public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ClusterSpec cluster) { - boolean exclusive = nodeRepository.exclusiveAllocation(cluster); + boolean exclusive = nodeRepository.exclusivity().allocation(cluster); if (! requested.vcpuIsUnspecified() && requested.vcpu() < minAdvertisedVcpu(cluster, exclusive)) illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(cluster, exclusive)); if (! requested.memoryGbIsUnspecified() && requested.memoryGb() < minAdvertisedMemoryGb(cluster, exclusive)) @@ -104,7 +105,7 @@ public class NodeResourceLimits { } private double minRealVcpu(ClusterSpec cluster) { - return minAdvertisedVcpu(cluster, nodeRepository.exclusiveAllocation(cluster)); + return minAdvertisedVcpu(cluster, nodeRepository.exclusivity().allocation(cluster)); } private static double minRealMemoryGb(ClusterSpec cluster) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 0206c3a4a26..ec5c807c9b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -202,7 +202,7 @@ public class Preparer { nodeRepository.nodes(), nodeRepository.resourcesCalculator(), nodeRepository.spareCount(), - nodeRepository.exclusiveAllocation(cluster)); + nodeRepository.exclusivity().allocation(cluster)); allocation.offer(prioritizer.collect()); if (requested.type() == NodeType.tenant && !requested.canFail() && allocation.changes()) { // This should not happen and indicates a bug in the allocation code because boostrap redeployment @@ -237,8 +237,8 @@ public class Preparer { private HostSharing hostSharing(ClusterSpec cluster, NodeType hostType) { if ( hostType.isSharable()) - return nodeRepository.exclusiveProvisioning(cluster) ? HostSharing.provision : - nodeRepository.exclusiveAllocation(cluster) ? HostSharing.exclusive : + return nodeRepository.exclusivity().provisioning(cluster) ? HostSharing.provision : + nodeRepository.exclusivity().allocation(cluster) ? HostSharing.exclusive : HostSharing.any; else return HostSharing.any; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 9d9771b3b0f..c085a3ed27c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; @@ -19,6 +20,7 @@ import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; import com.yahoo.config.provision.WireguardKeyWithTimestamp; @@ -90,6 +92,7 @@ public class MockNodeRepository extends NodeRepository { curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 79215028dba..fb96d6f798e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.test.ManualClock; @@ -40,16 +42,18 @@ public class NodeRepositoryTester { clock = new ManualClock(); curator = new MockCurator(); curator.setZooKeeperEnsembleConnectionSpec("server1:1234,server2:5678"); + var flagSource = new InMemoryFlagSource(); nodeRepository = new NodeRepository(nodeFlavors, new EmptyProvisionServiceProvider(), curator, clock, zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), Optional.empty(), - new InMemoryFlagSource(), + flagSource, new MemoryMetricsDb(clock), new OrchestratorMock(), true, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 8374c41ea25..e0c8199a882 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; @@ -162,7 +161,7 @@ public class AutoscalingTest { .awsProdSetup(false) .capacity(Capacity.from(min, max)) .initialResources(Optional.empty()) - .hostSharingFlag() + .hostSharing() .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 7, 1, 2.0, 10.0, 384.0, @@ -178,7 +177,7 @@ public class AutoscalingTest { .clusterType(ClusterSpec.Type.container) .capacity(Capacity.from(min, max)) .initialResources(Optional.empty()) - .hostSharingFlag() + .hostSharing() .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 1, 1, 0.5, 4.0, 10.0, @@ -389,7 +388,8 @@ public class AutoscalingTest { .build(); NodeResources defaultResources = - new CapacityPolicies(fixture.tester().nodeRepository()).specifyFully(NodeResources.unspecified(), fixture.clusterSpec, fixture.applicationId); + fixture.tester().nodeRepository().capacityPoliciesFor(fixture.applicationId) + .specifyFully(NodeResources.unspecified(), fixture.clusterSpec); fixture.tester().assertResources("Min number of nodes and default resources", 2, 1, defaultResources, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 4ce909fece3..f9f3ca9db8d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -11,12 +11,10 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.custom.HostResources; -import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.applications.Application; @@ -51,7 +49,7 @@ public class Fixture { applicationId = builder.application; clusterSpec = builder.cluster; capacity = builder.capacity; - tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); + tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, builder.sharedHosts, hostCount); var deployCapacity = initialResources.isPresent() ? Capacity.from(initialResources.get()) : capacity; tester.deploy(builder.application, builder.cluster, deployCapacity); this.loader = new Loader(this); @@ -178,6 +176,7 @@ public class Fixture { HostResourcesCalculator resourceCalculator = new DynamicProvisioningTester.MockHostResourcesCalculator(zone); final InMemoryFlagSource flagSource = new InMemoryFlagSource(); int hostCount = 0; + SharedHosts sharedHosts = SharedHosts.empty(); public Fixture.Builder zone(Zone zone) { this.zone = zone; @@ -283,9 +282,8 @@ public class Fixture { return this; } - public Fixture.Builder hostSharingFlag() { - var resources = new HostResources(8.0, 32.0, 100.0, 10.0, "fast", "local", null, 6, "x86_64"); - flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(resources)), SharedHost.class); + public Fixture.Builder hostSharing() { + sharedHosts = SharedHosts.ofConstant(true, false); return this; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index 1de8a766ea9..68303535f8c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.Exclusivity; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.json.Jackson; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -68,6 +70,7 @@ public class CapacityCheckerTester { curator, clock, zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index 9fefc9d34e1..dd43a27b0c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -7,12 +7,14 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; @@ -258,11 +260,13 @@ public class SpareCapacityMaintainerTest { private SpareCapacityMaintainerTester(int maxIterations) { NodeFlavors flavors = new NodeFlavors(new FlavorConfigBuilder().build()); ManualClock clock = new ManualClock(); + var zone = new Zone(Environment.prod, RegionName.from("us-east-3")); nodeRepository = new NodeRepository(flavors, new EmptyProvisionServiceProvider(), new MockCurator(), clock, - new Zone(Environment.prod, RegionName.from("us-east-3")), + zone, + new Exclusivity(zone, SharedHosts.empty()), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), Optional.empty(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 78a34326949..3a9d03ed6b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -207,9 +208,8 @@ public class DynamicProvisioningTest { private void assertHostSharing(Environment environment, ClusterSpec.Type clusterType, boolean expectShared) { Zone zone = new Zone(Cloud.builder().dynamicProvisioning(true).allowHostSharing(false).build(), SystemName.Public, environment, RegionName.defaultName()); MockHostProvisioner hostProvisioner = new MockHostProvisioner(new NodeFlavors(ProvisioningTester.createConfig()).getFlavors(), nameResolver, 0); - ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone).hostProvisioner(hostProvisioner).nameResolver(nameResolver).build(); + ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone).hostProvisioner(hostProvisioner).nameResolver(nameResolver).sharedHosts(SharedHosts.ofConstant(true, false)).build(); tester.makeReadyHosts(2, new NodeResources(12, 12, 200, 12)); - tester.flagSource().withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(new HostResources(4.0, 16.0, 50.0, 0.3, "fast", "local", null, 10, "x86_64"))), SharedHost.class); ApplicationId application = applicationId(); ClusterSpec cluster = ClusterSpec.request(clusterType, ClusterSpec.Id.from("default")).vespaVersion("6.42").build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 401b6d83651..7a6955de811 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -10,6 +11,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.Mutex; @@ -51,16 +53,15 @@ public class DynamicProvisioningTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; private final HostResourcesCalculator hostResourcesCalculator; - private final CapacityPolicies capacityPolicies; - public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { - this(zone, hostFlavors, resourcesCalculator, flagSource); + public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, SharedHosts sharedHosts, int hostCount) { + this(zone, hostFlavors, resourcesCalculator, flagSource, sharedHosts); for (Flavor flavor : hostFlavors) provisioningTester.makeReadyNodes(hostCount, flavor.name(), NodeType.host, 8); provisioningTester.activateTenantHosts(); } - private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { + private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource, SharedHosts sharedHosts) { MockHostProvisioner hostProvisioner = null; if (zone.cloud().dynamicProvisioning()) { hostProvisioner = new MockHostProvisioner(flavors); @@ -73,11 +74,11 @@ public class DynamicProvisioningTester { .resourcesCalculator(resourcesCalculator) .flagSource(flagSource) .hostProvisioner(hostProvisioner) + .sharedHosts(sharedHosts) .build(); hostResourcesCalculator = resourcesCalculator; autoscaler = new Autoscaler(nodeRepository()); - capacityPolicies = new CapacityPolicies(provisioningTester.nodeRepository()); } public InMemoryProvisionLogger provisionLogger() { return provisioningTester.provisionLogger(); } @@ -158,7 +159,8 @@ public class DynamicProvisioningTester { } public Autoscaling autoscale(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster).isExclusive()); + var capacityPolicies = provisioningTester.nodeRepository().capacityPoliciesFor(applicationId); + capacity = capacityPolicies.applyOn(capacity, capacityPolicies.decideExclusivity(capacity, cluster).isExclusive()); Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(cluster.id(), false, capacity); try (Mutex lock = nodeRepository().applications().lock(applicationId)) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 7b690b880c2..9e6ae9f010c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -46,12 +47,14 @@ import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import static java.time.temporal.ChronoUnit.MILLIS; +import static java.util.Objects.requireNonNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -1114,12 +1117,12 @@ public class ProvisioningTest { new Version("6"), new NodeResources(1, 1, 1, 1)); assertThrows(NullPointerException.class, - () -> CapacityPolicies.versioned(spec.vespaVersion("5.0").build(), resources)); - assertEquals(new NodeResources(1, 1, 1, 1), CapacityPolicies.versioned(spec.vespaVersion("6.0").build(), resources)); - assertEquals(new NodeResources(2, 2, 2, 2), CapacityPolicies.versioned(spec.vespaVersion("7.0").build(), resources)); - assertEquals(new NodeResources(2, 2, 2, 2), CapacityPolicies.versioned(spec.vespaVersion("7.1").build(), resources)); - assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("8.0").build(), resources)); - assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources)); + () -> versioned(spec.vespaVersion("5.0").build(), resources)); + assertEquals(new NodeResources(1, 1, 1, 1), versioned(spec.vespaVersion("6.0").build(), resources)); + assertEquals(new NodeResources(2, 2, 2, 2), versioned(spec.vespaVersion("7.0").build(), resources)); + assertEquals(new NodeResources(2, 2, 2, 2), versioned(spec.vespaVersion("7.1").build(), resources)); + assertEquals(new NodeResources(3, 3, 3, 3), versioned(spec.vespaVersion("8.0").build(), resources)); + assertEquals(new NodeResources(3, 3, 3, 3), versioned(spec.vespaVersion("9.0").build(), resources)); } @Test @@ -1292,4 +1295,13 @@ public class ProvisioningTest { return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); } + /** + * Returns the resources for the newest version not newer than that requested in the cluster spec. + */ + static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { + return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), + "no default resources applicable for " + spec + " among: " + resources) + .getValue(); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 4ec290dd7ba..390da7c719a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -8,12 +8,14 @@ import com.yahoo.config.provision.ApplicationMutex; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -24,6 +26,7 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SharedHosts; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -92,7 +95,6 @@ public class ProvisioningTester { private final NodeRepository nodeRepository; private final HostProvisioner hostProvisioner; private final NodeRepositoryProvisioner provisioner; - private final CapacityPolicies capacityPolicies; private final InMemoryProvisionLogger provisionLogger; private final LoadBalancerServiceMock loadBalancerService; @@ -110,7 +112,8 @@ public class ProvisioningTester { LoadBalancerServiceMock loadBalancerService, FlagSource flagSource, int spareCount, - ManualClock clock) { + ManualClock clock, + SharedHosts sharedHosts) { this.curator = curator; this.nodeFlavors = nodeFlavors; this.clock = clock; @@ -121,6 +124,7 @@ public class ProvisioningTester { curator, clock, zone, + new Exclusivity(zone, sharedHosts), nameResolver, containerImage, Optional.empty(), @@ -131,7 +135,6 @@ public class ProvisioningTester { true, spareCount); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, new MockMetric()); - this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new InMemoryProvisionLogger(); this.loadBalancerService = loadBalancerService; } @@ -159,14 +162,13 @@ public class ProvisioningTester { public NodeRepositoryProvisioner provisioner() { return provisioner; } public HostProvisioner hostProvisioner() { return hostProvisioner; } public LoadBalancerServiceMock loadBalancerService() { return loadBalancerService; } - public CapacityPolicies capacityPolicies() { return capacityPolicies; } public NodeList getNodes(ApplicationId id, Node.State ... inState) { return nodeRepository.nodes().list(inState).owner(id); } public InMemoryFlagSource flagSource() { return (InMemoryFlagSource) nodeRepository.flagSource(); } public InMemoryProvisionLogger provisionLogger() { return provisionLogger; } public Node node(String hostname) { return nodeRepository.nodes().node(hostname).get(); } public int decideSize(Capacity capacity, ApplicationId application) { - return capacityPolicies.applyOn(capacity, application, false).minResources().nodes(); + return nodeRepository.capacityPoliciesFor(application).applyOn(capacity, false).minResources().nodes(); } public Node patchNode(Node node, UnaryOperator<Node> patcher) { @@ -661,6 +663,7 @@ public class ProvisioningTester { private int spareCount = 0; private ManualClock clock = new ManualClock(); private DockerImage defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"); + private SharedHosts sharedHosts = SharedHosts.empty(); public Builder curator(Curator curator) { this.curator = curator; @@ -742,6 +745,11 @@ public class ProvisioningTester { return this; } + public Builder sharedHosts(SharedHosts sharedHosts) { + this.sharedHosts = sharedHosts; + return this; + } + private FlagSource defaultFlagSource() { return new InMemoryFlagSource(); } @@ -758,7 +766,8 @@ public class ProvisioningTester { new LoadBalancerServiceMock(), Optional.ofNullable(flagSource).orElse(defaultFlagSource()), spareCount, - clock); + clock, + sharedHosts); } private static FlavorsConfig asConfig(List<Flavor> flavors) { |