diff options
author | Jon Bratseth <bratseth@gmail.com> | 2023-03-01 20:28:23 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2023-03-01 20:28:23 +0100 |
commit | 495e73e2b3859eba3facd3915f7ac4b4a91bc66b (patch) | |
tree | f079858570367bb154206b4271a89c9fa96a80c1 /node-repository | |
parent | 8a2f4892657953379ed00a225916f128f0feec97 (diff) |
Use bcp deadline
Diffstat (limited to 'node-repository')
7 files changed, 77 insertions, 15 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index ea35f1e85ff..16016815b7c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; @@ -33,6 +34,7 @@ public class Cluster { private final boolean required; private final Autoscaling suggested; private final Autoscaling target; + private final ClusterInfo clusterInfo; private final BcpGroupInfo bcpGroupInfo; /** The maxScalingEvents last scaling events of this, sorted by increasing time (newest last) */ @@ -46,6 +48,7 @@ public class Cluster { boolean required, Autoscaling suggested, Autoscaling target, + ClusterInfo clusterInfo, BcpGroupInfo bcpGroupInfo, List<ScalingEvent> scalingEvents) { this.id = Objects.requireNonNull(id); @@ -60,6 +63,7 @@ public class Cluster { this.target = target.withResources(Optional.empty()); // Delete illegal target else this.target = target; + this.clusterInfo = clusterInfo; this.bcpGroupInfo = Objects.requireNonNull(bcpGroupInfo); this.scalingEvents = List.copyOf(scalingEvents); } @@ -105,6 +109,8 @@ public class Cluster { return true; } + public ClusterInfo clusterInfo() { return clusterInfo; } + /** Returns info about the BCP group of clusters this belongs to. */ public BcpGroupInfo bcpGroupInfo() { return bcpGroupInfo; } @@ -119,19 +125,19 @@ public class Cluster { public Cluster withConfiguration(boolean exclusive, Capacity capacity) { return new Cluster(id, exclusive, capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(), - suggested, target, bcpGroupInfo, scalingEvents); + suggested, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); } public Cluster withSuggested(Autoscaling suggested) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster withTarget(Autoscaling target) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster with(BcpGroupInfo bcpGroupInfo) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } /** Add or update (based on "at" time) a scaling event */ @@ -145,7 +151,7 @@ public class Cluster { scalingEvents.add(scalingEvent); prune(scalingEvents); - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } @Override @@ -177,7 +183,7 @@ public class Cluster { public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) { return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(), - Autoscaling.empty(), Autoscaling.empty(), BcpGroupInfo.empty(), List.of()); + Autoscaling.empty(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); } /** The predicted time it will take to rescale this cluster. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 3f916f1cc59..4e1523834e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -193,23 +193,28 @@ public class ClusterModel { /** * Returns the ideal load across the nodes of this such that each node will be at ideal load - * if one of the nodes go down. + * if one of the nodes go down. */ public Load idealLoad() { var ideal = new Load(idealCpuLoad(), idealMemoryLoad(), idealDiskLoad()).divide(redundancyAdjustment()); - if (! cluster.bcpGroupInfo().isEmpty()) { + if ( !cluster.bcpGroupInfo().isEmpty() && cluster.bcpGroupInfo().queryRate() > 0) { + // Since we have little local information, use information about query cost in other groups + + Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal); + // Do a weighted sum of the ideal "vote" based on local and bcp group info. // This avoids any discontinuities with a near-zero local query rate. - double localInformationWeight = cluster.bcpGroupInfo().queryRate() == 0 - ? 1 - : Math.min(1, averageQueryRate().orElse(0) / - Math.min(queryRateGivingFullConfidence, cluster.bcpGroupInfo().queryRate())); - Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal); + double localInformationWeight = Math.min(1, averageQueryRate().orElse(0) / + Math.min(queryRateGivingFullConfidence, cluster.bcpGroupInfo().queryRate())); ideal = ideal.multiply(localInformationWeight).add(bcpGroupIdeal.multiply(1 - localInformationWeight)); } return ideal; } + private boolean canRescaleWithinBcpDeadline() { + return scalingDuration().minus(cluster.clusterInfo().bcpDeadline()).isNegative(); + } + public Autoscaling.Metrics metrics() { return new Autoscaling.Metrics(averageQueryRate().orElse(0), growthRateHeadroom(), @@ -230,7 +235,9 @@ public class ClusterModel { private Load adjustQueryDependentIdealLoadByBcpGroupInfo(Load ideal) { double currentClusterTotalVcpuPerGroup = nodes.not().retired().first().get().resources().vcpu() * groupSize(); - double targetQueryRateToHandle = cluster.bcpGroupInfo().queryRate() * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); + double targetQueryRateToHandle = ( canRescaleWithinBcpDeadline() ? averageQueryRate().orElse(0) + : cluster.bcpGroupInfo().queryRate() ) + * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); double neededTotalVcpPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() + ( 1 - queryCpuFraction()) * idealCpuLoad() * (clusterSpec.type().isContainer() ? 1 : groupSize()); @@ -325,6 +332,7 @@ public class ClusterModel { */ private double trafficShiftHeadroom() { if ( ! zone.environment().isProduction()) return 1; + if (canRescaleWithinBcpDeadline()) return 1; double trafficShiftHeadroom; if (application.status().maxReadShare() == 0) // No traffic fraction data trafficShiftHeadroom = 2.0; // assume we currently get half of the max possible share of traffic diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 38675ba3758..c1126b769e6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Load; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -54,6 +56,8 @@ public class ApplicationSerializer { private static final String groupSizeKey = "groupSize"; private static final String requiredKey = "required"; private static final String suggestedKey = "suggested"; + private static final String clusterInfoKey = "clusterInfo"; + private static final String bcpDeadlineKey = "clusterInfo"; private static final String bcpGroupInfoKey = "bcpGroupInfo"; private static final String queryRateKey = "queryRateKey"; private static final String growthRateHeadroomKey = "growthRateHeadroomKey"; @@ -134,6 +138,8 @@ public class ApplicationSerializer { clusterObject.setBool(requiredKey, cluster.required()); toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey)); toSlime(cluster.target(), clusterObject.setObject(targetKey)); + if (! cluster.clusterInfo().isEmpty()) + toSlime(cluster.clusterInfo(), clusterObject.setObject(clusterInfoKey)); if (! cluster.bcpGroupInfo().isEmpty()) toSlime(cluster.bcpGroupInfo(), clusterObject.setObject(bcpGroupInfoKey)); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); @@ -148,6 +154,7 @@ public class ApplicationSerializer { clusterObject.field(requiredKey).asBool(), autoscalingFromSlime(clusterObject.field(suggestedKey)), autoscalingFromSlime(clusterObject.field(targetKey)), + clusterInfoFromSlime(clusterObject.field(clusterInfoKey)), bcpGroupInfoFromSlime(clusterObject.field(bcpGroupInfoKey)), scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); } @@ -225,8 +232,18 @@ public class ApplicationSerializer { metricsFromSlime(autoscalingObject.field(metricsKey))); } + private static void toSlime(ClusterInfo clusterInfo, Cursor clusterInfoObject) { + clusterInfoObject.setLong(bcpDeadlineKey, clusterInfo.bcpDeadline().toMinutes()); + } + + private static ClusterInfo clusterInfoFromSlime(Inspector clusterInfoObject) { + if ( ! clusterInfoObject.valid()) return ClusterInfo.empty(); + ClusterInfo.Builder builder = new ClusterInfo.Builder(); + builder.bcpDeadline(Duration.ofMinutes(clusterInfoObject.field(bcpDeadlineKey).asLong())); + return builder.build(); + } + private static void toSlime(BcpGroupInfo bcpGroupInfo, Cursor bcpGroupInfoObject) { - if (bcpGroupInfo.isEmpty()) return; bcpGroupInfoObject.setDouble(queryRateKey, bcpGroupInfo.queryRate()); bcpGroupInfoObject.setDouble(growthRateHeadroomKey, bcpGroupInfo.growthRateHeadroom()); bcpGroupInfoObject.setDouble(cpuCostPerQueryKey, bcpGroupInfo.cpuCostPerQuery()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java index bd3589be9dd..704491ed44f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java @@ -1,8 +1,10 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo; import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; @@ -154,6 +156,28 @@ public class AutoscalingUsingBcpGroupInfoTest { } @Test + public void test_autoscaling_with_bcp_deadline() { + var capacity = Capacity.from(new ClusterResources(2, 1, + new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)), + new ClusterResources(20, 1, + new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)), + IntRange.empty(), false, true, Optional.empty(), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(60)).build()); + + var fixture = DynamicProvisioningTester.fixture() + .capacity(capacity) + .clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); + + // We can rescale within deadline - do not take BCP info into account + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.tester().assertResources("No need for traffic shift headroom", + 2, 1, 2.0, 16.0, 40.8, + fixture.autoscale()); + } + + @Test public void test_autoscaling_single_content_group_with_some_local_traffic() { var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index f9dd4578f7a..48db3fade95 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index d04fe1bdda2..c429f88cfa1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.Load; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -40,6 +42,7 @@ public class ApplicationSerializerTest { true, Autoscaling.empty(), Autoscaling.empty(), + ClusterInfo.empty(), BcpGroupInfo.empty(), List.of())); var minResources = new NodeResources(1, 2, 3, 4); @@ -65,6 +68,7 @@ public class ApplicationSerializerTest { Load.zero(), Load.one(), Autoscaling.Metrics.zero()), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(33)).build(), new BcpGroupInfo(0.1, 0.2, 0.3), List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), new ClusterResources(12, 6, minResources), @@ -95,6 +99,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.required(), serializedCluster.required()); assertEquals(originalCluster.suggested(), serializedCluster.suggested()); assertEquals(originalCluster.target(), serializedCluster.target()); + assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo()); assertEquals(originalCluster.bcpGroupInfo(), serializedCluster.bcpGroupInfo()); assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 5beb4b11c0c..a24eb61bb79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -153,6 +153,7 @@ public class DynamicProvisioningTester { cluster.required(), cluster.suggested(), cluster.target(), + cluster.clusterInfo(), cluster.bcpGroupInfo(), List.of()); // Remove scaling events cluster = cluster.with(ScalingEvent.create(cluster.minResources(), cluster.minResources(), |