aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHarald Musum <musum@verizonmedia.com>2023-02-21 12:32:24 +0100
committerGitHub <noreply@github.com>2023-02-21 12:32:24 +0100
commit02707baee5757c8299020c7135826e3c2380c315 (patch)
tree4bbbd7a3f43757d8edfa365d2356972d79b0857f
parent722b67238df3c6e908512575b7cea52a5dd8d187 (diff)
parent901a1baf0f622d78937d8953d89acad759de1f9a (diff)
Merge pull request #26124 from vespa-engine/bratseth/admin-provisioning
Bratseth/admin provisioning
-rw-r--r--document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java2
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java8
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java7
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java36
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java11
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java303
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java17
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java11
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java1
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java6
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java4
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java (renamed from node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java)29
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java13
13 files changed, 255 insertions, 193 deletions
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
index b7a3589a4d0..4c4d9c78c8e 100644
--- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -775,7 +775,7 @@ public class DocumentSelectorTestCase {
@Test
public void testInheritance() throws ParseException {
- var s=new DocumentSelector("parent.parentField = \"parentValue\"");
+ new DocumentSelector("parent.parentField = \"parentValue\"");
List<DocumentPut> documents = createDocs();
assertEquals(Result.TRUE, evaluate("test", documents.get(0)));
assertEquals("Matching on type is exact", Result.FALSE, evaluate("parent", documents.get(0)));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 8af1df93bde..5732e94956a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -115,10 +115,12 @@ public class CapacityPolicies {
if (nodeRepository.exclusiveAllocation(clusterSpec)) {
return versioned(clusterSpec, Map.of(new Version(0), smallestExclusiveResources()));
}
- // TODO (hmusum): Go back to 1.14 Gb memory when bug in resource limits for admin nodes
- // has been fixed
+
+ // 1.32 fits floor(8/1.32) = 6 cluster controllers on each 8Gb host, and each will have
+ // 1.32-(0.7+0.6)*(1.32/8) = 1.1 Gb real memory given current taxes.
return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.14, 10, 0.3),
- new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3)));
+ new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3),
+ new Version(8, 129, 4), new NodeResources(0.25, 1.32, 10, 0.3)));
}
private Architecture adminClusterArchitecture(ApplicationId instance) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
index 1acb69113c1..4d33e1c7bad 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
@@ -88,10 +88,13 @@ public class NodeResourceLimits {
return 4;
}
- private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) { return minAdvertisedVcpu(applicationId, cluster); }
+ private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) {
+ return minAdvertisedVcpu(applicationId, cluster);
+ }
private double minRealMemoryGb(ClusterSpec cluster) {
- return minAdvertisedMemoryGb(cluster) - 1.7;
+ if (cluster.type() == ClusterSpec.Type.admin) return 0.95; // TODO: Increase to 1.05 after March 2023
+ return 2.3;
}
private double minRealDiskGb() { return 6; }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
index 84856ab310b..ad53dc8727d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.node.IP;
import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig;
import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
import java.time.Instant;
@@ -46,7 +47,7 @@ public class MockHostProvisioner implements HostProvisioner {
private int deprovisionedHosts = 0;
private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class);
- private Optional<Flavor> hostFlavor = Optional.empty();
+ private Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>();
public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb) {
this.flavors = List.copyOf(flavors);
@@ -67,11 +68,14 @@ public class MockHostProvisioner implements HostProvisioner {
ApplicationId applicationId, Version osVersion, HostSharing sharing,
Optional<ClusterSpec.Type> clusterType, CloudAccount cloudAccount,
Consumer<List<ProvisionedHost>> provisionedHostsConsumer) {
- Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream()
- .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources)
- : f.resources().satisfies(resources))
- .findFirst()
- .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true)));
+ Flavor hostFlavor = hostFlavors.get(clusterType.orElse(ClusterSpec.Type.content));
+ if (hostFlavor == null)
+ hostFlavor = flavors.stream()
+ .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources)
+ : f.resources().satisfies(resources))
+ .findFirst()
+ .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true));
+
List<ProvisionedHost> hosts = new ArrayList<>();
for (int index : provisionIndices) {
String hostHostname = hostType == NodeType.host ? "host" + index : hostType.name() + index;
@@ -152,14 +156,30 @@ public class MockHostProvisioner implements HostProvisioner {
return this;
}
- public MockHostProvisioner overrideHostFlavor(String flavorName) {
+ public MockHostProvisioner setHostFlavor(String flavorName, ClusterSpec.Type ... types) {
Flavor flavor = flavors.stream().filter(f -> f.name().equals(flavorName))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("No such flavor '" + flavorName + "'"));
- hostFlavor = Optional.of(flavor);
+ if (types.length == 0)
+ types = ClusterSpec.Type.values();
+ for (var type : types)
+ hostFlavors.put(type, flavor);
return this;
}
+ /** Sets the host flavor to use to the flavor matching these resources exactly, if any. */
+ public MockHostProvisioner setHostFlavorIfAvailable(NodeResources flavorAdvertisedResources, HostResourcesCalculator calculator, ClusterSpec.Type ... types) {
+ Optional<Flavor> hostFlavor = flavors.stream().filter(f -> calculator.advertisedResourcesOf(f).compatibleWith(flavorAdvertisedResources))
+ .findFirst();
+ if (types.length == 0)
+ types = ClusterSpec.Type.values();
+ for (var type : types)
+ hostFlavor.ifPresent(f -> hostFlavors.put(type, f));
+ return this;
+ }
+
+ public Optional<Flavor> getHostFlavor(ClusterSpec.Type type) { return Optional.ofNullable(hostFlavors.get(type)); }
+
public MockHostProvisioner addEvent(HostEvent event) {
hostEvents.add(event);
return this;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
index 158c5116e19..3ee72c18318 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.autoscale;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester;
import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
import org.junit.Test;
@@ -22,12 +23,12 @@ public class AutoscalingIntegrationTest {
@Test
public void testComponentIntegration() {
- var fixture = AutoscalingTester.fixture()
- .hostCount(20)
- .hostFlavors(new NodeResources(3, 20, 200, 1))
- .initialResources(Optional.of(new ClusterResources(2, 1,
+ var fixture = DynamicProvisioningTester.fixture()
+ .hostCount(20)
+ .hostFlavors(new NodeResources(3, 20, 200, 1))
+ .initialResources(Optional.of(new ClusterResources(2, 1,
new NodeResources(1, 10, 100, 1))))
- .build();
+ .build();
MetricsV2MetricsFetcher fetcher = new MetricsV2MetricsFetcher(fixture.tester().nodeRepository(),
new OrchestratorMock(),
new MockHttpClient(fixture.tester().clock()));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
index c19db34691a..11bc26d90a5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.NodeResources.StorageType;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.Zone;
import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies;
+import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester;
import org.junit.Test;
import java.time.Duration;
@@ -34,12 +35,12 @@ public class AutoscalingTest {
var min = new ClusterResources( 8, 1, resources);
var now = new ClusterResources(12, 1, resources.with(StorageType.remote));
var max = new ClusterResources(12, 1, resources);
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .clusterType(ClusterSpec.Type.content)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .clusterType(ClusterSpec.Type.content)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester.clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(0.17f, 0.17, 0.12), 1, true, true, 100);
var result = fixture.autoscale();
@@ -55,7 +56,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_single_content_group() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.loader().applyCpuLoad(0.7f, 10);
var scaledResources = fixture.tester().assertResources("Scaling up since resource usage is too high",
@@ -83,20 +84,20 @@ public class AutoscalingTest {
/** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */
@Test
public void test_no_autoscaling_with_no_measurements() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
assertTrue(fixture.autoscale().resources().isEmpty());
}
@Test
public void test_no_autoscaling_with_no_measurements_exclusive() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(false).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build();
assertTrue(fixture.autoscale().resources().isEmpty());
}
/** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */
@Test
public void test_autoscaling_up_is_fast() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.loader().applyLoad(new Load(0.1, 0.1, 0.1), 3);
fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 1);
fixture.tester().assertResources("Scaling up since resource usage is too high",
@@ -109,12 +110,12 @@ public class AutoscalingTest {
var min = new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1));
var now = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1));
var max = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(false)
- .clusterType(ClusterSpec.Type.container)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(false)
+ .clusterType(ClusterSpec.Type.container)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().setScalingDuration(fixture.applicationId(), fixture.clusterSpec.id(), Duration.ofMinutes(5));
fixture.loader().applyLoad(new Load(0.01, 0.38, 0), 5);
@@ -127,12 +128,12 @@ public class AutoscalingTest {
public void initial_deployment_with_host_sharing_flag() {
var min = new ClusterResources(7, 1, new NodeResources(2.0, 10.0, 384.0, 0.1));
var max = new ClusterResources(7, 1, new NodeResources(2.4, 32.0, 768.0, 0.1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(false)
- .capacity(Capacity.from(min, max))
- .initialResources(Optional.empty())
- .hostSharingFlag()
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(false)
+ .capacity(Capacity.from(min, max))
+ .initialResources(Optional.empty())
+ .hostSharingFlag()
+ .build();
fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing",
7, 1, 2.0, 10.0, 384.0,
fixture.currentResources().advertisedResources());
@@ -142,13 +143,13 @@ public class AutoscalingTest {
public void initial_deployment_with_host_sharing_flag_and_too_small_min() {
var min = new ClusterResources(1, 1, new NodeResources(0.5, 4.0, 10, 0.1));
var max = new ClusterResources(1, 1, new NodeResources(2.0, 8.0, 50, 0.1));
- var fixture = AutoscalingTester.fixture()
- .awsSetup(false, Environment.test)
- .clusterType(ClusterSpec.Type.container)
- .capacity(Capacity.from(min, max))
- .initialResources(Optional.empty())
- .hostSharingFlag()
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsSetup(false, Environment.test)
+ .clusterType(ClusterSpec.Type.container)
+ .capacity(Capacity.from(min, max))
+ .initialResources(Optional.empty())
+ .hostSharingFlag()
+ .build();
fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing",
1, 1, 0.5, 4.0, 10.0,
fixture.currentResources().advertisedResources());
@@ -157,7 +158,7 @@ public class AutoscalingTest {
/** When scaling up, disregard underutilized dimensions (memory here) */
@Test
public void test_only_autoscaling_up_quickly() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10);
fixture.tester().assertResources("Scaling up (only) since resource usage is too high",
8, 1, 7.1, 8.8, 75.4,
@@ -167,7 +168,7 @@ public class AutoscalingTest {
/** When ok to scale down, scale in both directions simultaneously (compare to test_only_autoscaling_up_quickly) */
@Test
public void test_scale_in_both_directions_when_ok_to_scale_down() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.tester.clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10);
fixture.tester().assertResources("Scaling cpu and disk up and memory down",
@@ -177,7 +178,7 @@ public class AutoscalingTest {
@Test
public void test_scale_in_both_directions_when_ok_to_scale_down_exclusive() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(false).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build();
fixture.tester.clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10);
fixture.tester().assertResources("Scaling cpu and disk up, memory follows",
@@ -187,7 +188,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_uses_peak() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.loader().applyCpuLoad(0.01, 100);
fixture.loader().applyCpuLoad(0.70, 1);
fixture.loader().applyCpuLoad(0.01, 100);
@@ -198,7 +199,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_uses_peak_exclusive() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(false).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build();
fixture.loader().applyCpuLoad(0.01, 100);
fixture.loader().applyCpuLoad(0.70, 1);
fixture.loader().applyCpuLoad(0.01, 100);
@@ -209,7 +210,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_uses_peak_preprovisioned() {
- var fixture = AutoscalingTester.fixture().hostCount(15).build();
+ var fixture = DynamicProvisioningTester.fixture().hostCount(15).build();
fixture.loader().applyCpuLoad(0.01, 100);
fixture.loader().applyCpuLoad(0.70, 1);
fixture.loader().applyCpuLoad(0.01, 100);
@@ -223,10 +224,10 @@ public class AutoscalingTest {
var min = new ClusterResources(1, 1, new NodeResources(0.5, 4, 10, 0.3));
var now = new ClusterResources(4, 1, new NodeResources(8, 16, 10, 0.3));
var max = new ClusterResources(4, 1, new NodeResources(16, 32, 50, 0.3));
- var fixture = AutoscalingTester.fixture(min, now, max)
- .clusterType(ClusterSpec.Type.container)
- .awsProdSetup(false)
- .build();
+ var fixture = DynamicProvisioningTester.fixture(min, now, max)
+ .clusterType(ClusterSpec.Type.container)
+ .awsProdSetup(false)
+ .build();
var duration = fixture.loader().addMeasurements(new Load(0.04, 0.39, 0.01), 20);
fixture.tester().clock().advance(duration.negated());
fixture.loader().zeroTraffic(20, 1);
@@ -238,7 +239,7 @@ public class AutoscalingTest {
/** We prefer fewer nodes for container clusters as (we assume) they all use the same disk and memory */
@Test
public void test_autoscaling_single_container_group() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).clusterType(ClusterSpec.Type.container).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).clusterType(ClusterSpec.Type.container).build();
fixture.loader().applyCpuLoad(0.25f, 120);
ClusterResources scaledResources = fixture.tester().assertResources("Scaling cpu up",
@@ -255,12 +256,12 @@ public class AutoscalingTest {
@Test
public void autoscaling_handles_disk_setting_changes_exclusive_preprovisioned() {
var resources = new NodeResources(3, 100, 100, 1, slow);
- var fixture = AutoscalingTester.fixture()
- .hostCount(20)
- .hostFlavors(resources)
- .initialResources(Optional.of(new ClusterResources(5, 1, resources)))
- .capacity(Capacity.from(new ClusterResources(5, 1, resources)))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .hostCount(20)
+ .hostFlavors(resources)
+ .initialResources(Optional.of(new ClusterResources(5, 1, resources)))
+ .capacity(Capacity.from(new ClusterResources(5, 1, resources)))
+ .build();
assertTrue(fixture.tester().nodeRepository().nodes().list().owner(fixture.applicationId).stream()
.allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == slow));
@@ -289,11 +290,11 @@ public class AutoscalingTest {
NodeResources resources = new NodeResources(1, 100, 100, 1);
var capacity = Capacity.from(new ClusterResources( 2, 1, resources.with(DiskSpeed.any)),
new ClusterResources( 10, 1, resources.with(DiskSpeed.any)));
- var fixture = AutoscalingTester.fixture()
- .capacity(capacity)
- .awsProdSetup(true)
- .initialResources(Optional.empty())
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .capacity(capacity)
+ .awsProdSetup(true)
+ .initialResources(Optional.empty())
+ .build();
// Redeployment without target: Uses current resource numbers with *requested* non-numbers (i.e disk-speed any)
assertTrue(fixture.tester().nodeRepository().applications().get(fixture.applicationId).get().cluster(fixture.clusterSpec.id()).get().target().resources().isEmpty());
@@ -313,10 +314,10 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(5, 1, new NodeResources(1.9, 70, 70, 1));
var max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max)).build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max)).build();
fixture.tester().clock().advance(Duration.ofDays(1));
fixture.loader().applyLoad(new Load(0.25, 0.95, 0.95), 120);
@@ -329,7 +330,7 @@ public class AutoscalingTest {
public void autoscaling_respects_lower_limit() {
var min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1));
var max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1));
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, max)).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, max)).build();
// deploy
fixture.tester().clock().advance(Duration.ofDays(2));
@@ -343,11 +344,11 @@ public class AutoscalingTest {
public void autoscaling_with_unspecified_resources_use_defaults_exclusive() {
var min = new ClusterResources( 2, 1, NodeResources.unspecified());
var max = new ClusterResources( 6, 1, NodeResources.unspecified());
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(false)
- .initialResources(Optional.empty())
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(false)
+ .initialResources(Optional.empty())
+ .capacity(Capacity.from(min, max))
+ .build();
NodeResources defaultResources =
new CapacityPolicies(fixture.tester().nodeRepository()).defaultNodeResources(fixture.clusterSpec, fixture.applicationId);
@@ -368,11 +369,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(5, 5, new NodeResources(3.0, 10, 10, 1));
var max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyCpuLoad(0.4, 240);
fixture.tester().assertResources("Scaling cpu up",
@@ -385,11 +386,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(5, 5, new NodeResources(3.0, 10, 10, 1));
var max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty()))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty()))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyCpuLoad(0.4, 240);
fixture.tester().assertResources("Scaling cpu up",
@@ -400,7 +401,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_limits_when_min_equals_max() {
ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build();
fixture.tester().clock().advance(Duration.ofDays(1));
fixture.loader().applyCpuLoad(0.25, 120);
@@ -412,14 +413,14 @@ public class AutoscalingTest {
var resources = new ClusterResources( 2, 1, new NodeResources(3, 100, 50, 1));
var local = new NodeResources(3, 100, 75, 1, fast, StorageType.local);
var remote = new NodeResources(3, 100, 50, 1, fast, StorageType.remote);
- var fixture = AutoscalingTester.fixture()
- .dynamicProvisioning(true)
- .allowHostSharing(false)
- .clusterType(ClusterSpec.Type.container)
- .hostFlavors(local, remote)
- .capacity(Capacity.from(resources))
- .initialResources(Optional.of(new ClusterResources(3, 1, resources.nodeResources())))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .dynamicProvisioning(true)
+ .allowHostSharing(false)
+ .clusterType(ClusterSpec.Type.container)
+ .hostFlavors(local, remote)
+ .capacity(Capacity.from(resources))
+ .initialResources(Optional.of(new ClusterResources(3, 1, resources.nodeResources())))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(0.01, 0.01, 0.01), 120);
@@ -434,7 +435,7 @@ public class AutoscalingTest {
@Test
public void suggestions_ignores_limits() {
ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyCpuLoad(1.0, 120);
fixture.tester().assertResources("Suggesting above capacity limit",
@@ -445,7 +446,7 @@ public class AutoscalingTest {
@Test
public void suggestions_ignores_limits_exclusive() {
ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
- var fixture = AutoscalingTester.fixture().awsProdSetup(false).capacity(Capacity.from(min, min)).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).capacity(Capacity.from(min, min)).build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyCpuLoad(1.0, 120);
fixture.tester().assertResources("Suggesting above capacity limit",
@@ -455,7 +456,7 @@ public class AutoscalingTest {
@Test
public void not_using_out_of_service_measurements() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(0.9, 0.6, 0.7), 1, false, true, 120);
assertTrue("Not scaling up since nodes were measured while cluster was out of service",
@@ -464,7 +465,7 @@ public class AutoscalingTest {
@Test
public void not_using_unstable_measurements() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(0.9, 0.6, 0.7), 1, true, false, 120);
assertTrue("Not scaling up since nodes were measured while cluster was unstable",
@@ -476,11 +477,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(5, 5, new NodeResources(3, 100, 100, 1));
var max = new ClusterResources(20, 20, new NodeResources(10, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyCpuLoad(0.9, 120);
fixture.tester().assertResources("Scaling up to 2 nodes, scaling memory and disk down at the same time",
@@ -493,11 +494,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(5, 5, new NodeResources(3, 100, 100, 1));
var max = new ClusterResources(20, 20, new NodeResources(10, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty()))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty()))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyCpuLoad(0.9, 120);
fixture.tester().assertResources("Scaling up to 2 nodes, scaling memory and disk down at the same time",
@@ -510,11 +511,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(6, 2, new NodeResources(3, 100, 100, 1));
var max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120);
fixture.tester().clock().advance(timePassed.negated());
@@ -530,11 +531,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(6, 2, new NodeResources(3, 100, 100, 1));
var max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120);
fixture.tester().clock().advance(timePassed.negated());
@@ -549,11 +550,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(6, 2, new NodeResources(10, 100, 100, 1));
var max = new ClusterResources(30, 30, new NodeResources(100, 100, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(1));
fixture.loader().applyMemLoad(1.0, 1000);
fixture.tester().assertResources("Increase group size to reduce memory load",
@@ -566,11 +567,11 @@ public class AutoscalingTest {
var min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
var now = new ClusterResources(6, 1, new NodeResources(3, 100, 100, 1));
var max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(now))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(now))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(0.16, 0.02, 0.5), 120);
fixture.tester().assertResources("Scaling down memory",
@@ -580,7 +581,7 @@ public class AutoscalingTest {
@Test
public void scaling_down_only_after_delay() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.loader().applyCpuLoad(0.02, 120);
assertTrue("Too soon after initial deployment", fixture.autoscale().resources().isEmpty());
fixture.tester().clock().advance(Duration.ofDays(2));
@@ -594,10 +595,10 @@ public class AutoscalingTest {
public void test_autoscaling_considers_read_share() {
var min = new ClusterResources( 1, 1, new NodeResources(3, 100, 100, 1));
var max = new ClusterResources(10, 1, new NodeResources(3, 100, 100, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester.clock().advance(Duration.ofDays(1));
fixture.loader().applyCpuLoad(0.25, 120);
@@ -622,7 +623,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_considers_growth_rate() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.tester().clock().advance(Duration.ofDays(2));
Duration timeAdded = fixture.loader().addLoadMeasurements(100, t -> t == 0 ? 200.0 : 100.0, t -> 0.0);
@@ -656,7 +657,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_weights_growth_rate_by_confidence() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
double scalingFactor = 1.0/6000; // To make the average query rate low
fixture.setScalingDuration(Duration.ofMinutes(60));
@@ -673,7 +674,7 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_considers_query_vs_write_rate() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.loader().addCpuMeasurements(0.4, 220);
@@ -724,10 +725,10 @@ public class AutoscalingTest {
@Test
public void test_autoscaling_in_dev_preprovisioned() {
- var fixture = AutoscalingTester.fixture()
- .hostCount(5)
- .zone(new Zone(Environment.dev, RegionName.from("us-east")))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .hostCount(5)
+ .zone(new Zone(Environment.dev, RegionName.from("us-east")))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200);
assertTrue("Not attempting to scale up because policies dictate we'll only get one node",
@@ -740,10 +741,10 @@ public class AutoscalingTest {
new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any));
var max = new ClusterResources(20, 20,
new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any));
- var fixture = AutoscalingTester.fixture()
- .awsSetup(true, Environment.dev)
- .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty()))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsSetup(true, Environment.dev)
+ .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty()))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200);
fixture.tester().assertResources("Scale only to a single node and group since this is dev",
@@ -764,11 +765,11 @@ public class AutoscalingTest {
true,
Optional.empty());
- var fixture = AutoscalingTester.fixture()
- .hostCount(5)
- .capacity(requiredCapacity)
- .zone(new Zone(Environment.dev, RegionName.from("us-east")))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .hostCount(5)
+ .capacity(requiredCapacity)
+ .zone(new Zone(Environment.dev, RegionName.from("us-east")))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200);
fixture.tester().assertResources("We scale even in dev because resources are 'required'",
@@ -786,11 +787,11 @@ public class AutoscalingTest {
true,
Optional.empty());
- var fixture = AutoscalingTester.fixture()
- .hostCount(5)
- .capacity(requiredCapacity)
- .zone(new Zone(Environment.dev, RegionName.from("us-east")))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .hostCount(5)
+ .capacity(requiredCapacity)
+ .zone(new Zone(Environment.dev, RegionName.from("us-east")))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200);
fixture.tester().assertResources("We scale even in dev because resources are required",
@@ -802,12 +803,12 @@ public class AutoscalingTest {
public void test_changing_exclusivity() {
var min = new ClusterResources( 2, 1, new NodeResources( 3, 4, 100, 1));
var max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .cluster(clusterSpec(true))
- .capacity(Capacity.from(min, max))
- .initialResources(Optional.empty())
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .cluster(clusterSpec(true))
+ .capacity(Capacity.from(min, max))
+ .initialResources(Optional.empty())
+ .build();
fixture.tester().assertResources("Initial deployment at minimum",
2, 1, 4, 8, 100,
fixture.currentResources().advertisedResources());
@@ -831,11 +832,11 @@ public class AutoscalingTest {
var min = new ClusterResources(7, 1, new NodeResources( 2, 10, 384, 1));
var now = new ClusterResources(7, 1, new NodeResources( 3.4, 16.2, 450.1, 1));
var max = new ClusterResources(7, 1, new NodeResources( 4, 32, 768, 1));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .capacity(Capacity.from(min, max))
- .initialResources(Optional.of(now))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .capacity(Capacity.from(min, max))
+ .initialResources(Optional.of(now))
+ .build();
var initialNodes = fixture.nodes().asList();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.loader().applyLoad(new Load(0.06, 0.52, 0.27), 100);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java
index eb2488b7829..91de7209ac1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java
@@ -5,6 +5,7 @@ import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo;
+import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester;
import org.junit.Test;
import java.time.Duration;
@@ -21,7 +22,7 @@ public class AutoscalingUsingBcpGroupInfoTest {
/** Tests with varying BCP group info parameters. */
@Test
public void test_autoscaling_single_content_group() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.store(new BcpGroupInfo(100, 1.1, 0.3));
@@ -62,11 +63,11 @@ public class AutoscalingUsingBcpGroupInfoTest {
new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any));
var max = new ClusterResources(21, 3,
new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any));
- var fixture = AutoscalingTester.fixture()
- .awsProdSetup(true)
- .initialResources(Optional.of(new ClusterResources(9, 3, new NodeResources(2, 16, 75, 1))))
- .capacity(Capacity.from(min, max))
- .build();
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .initialResources(Optional.of(new ClusterResources(9, 3, new NodeResources(2, 16, 75, 1))))
+ .capacity(Capacity.from(min, max))
+ .build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.store(new BcpGroupInfo(100, 1.1, 0.3));
@@ -108,7 +109,7 @@ public class AutoscalingUsingBcpGroupInfoTest {
*/
@Test
public void test_autoscaling_container() {
- var fixture = AutoscalingTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build();
fixture.tester().clock().advance(Duration.ofDays(2));
fixture.store(new BcpGroupInfo(100, 1.1, 0.3));
@@ -144,7 +145,7 @@ public class AutoscalingUsingBcpGroupInfoTest {
@Test
public void test_autoscaling_single_content_group_with_some_local_traffic() {
- var fixture = AutoscalingTester.fixture().awsProdSetup(true).build();
+ var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
// Baseline: No local traffic, group traffic indicates much higher cpu usage than local
fixture.tester().clock().advance(Duration.ofDays(2));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
index 5caf50a4e83..f9dd4578f7a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster;
import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo;
import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsHostResourcesCalculatorImpl;
import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsNodeTypes;
+import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester;
import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
import java.time.Duration;
import java.util.Arrays;
@@ -37,7 +38,7 @@ import java.util.Optional;
*/
public class Fixture {
- final AutoscalingTester tester;
+ final DynamicProvisioningTester tester;
final Zone zone;
final ApplicationId applicationId;
final ClusterSpec clusterSpec;
@@ -49,13 +50,13 @@ public class Fixture {
applicationId = builder.application;
clusterSpec = builder.cluster;
capacity = builder.capacity;
- tester = new AutoscalingTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount);
+ tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount);
var deployCapacity = initialResources.isPresent() ? Capacity.from(initialResources.get()) : capacity;
tester.deploy(builder.application, builder.cluster, deployCapacity);
this.loader = new Loader(this);
}
- public AutoscalingTester tester() { return tester; }
+ public DynamicProvisioningTester tester() { return tester; }
public ApplicationId applicationId() { return applicationId; }
@@ -141,7 +142,7 @@ public class Fixture {
public static class Builder {
- ApplicationId application = AutoscalingTester.applicationId("application1");
+ ApplicationId application = DynamicProvisioningTester.applicationId("application1");
ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster1")).vespaVersion("7").build();
Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
List<Flavor> hostFlavors = List.of(new Flavor(new NodeResources(100, 100, 100, 1)));
@@ -150,7 +151,7 @@ public class Fixture {
new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)),
new ClusterResources(20, 1,
new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)));
- HostResourcesCalculator resourceCalculator = new AutoscalingTester.MockHostResourcesCalculator(zone);
+ HostResourcesCalculator resourceCalculator = new DynamicProvisioningTester.MockHostResourcesCalculator(zone);
final InMemoryFlagSource flagSource = new InMemoryFlagSource();
int hostCount = 0;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java
index 96fa143dc57..69469bb03c7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java
@@ -48,6 +48,7 @@ public class AwsResourcesCalculator {
( hostFlavor.advertisedResources().memoryGb() - ( real ? hostMemoryOverhead : 0));
if (memoryShare > 1) // The real resources of the host cannot fit the requested real resources after overhead
memoryShare = 1;
+
return hostMemoryOverhead * memoryShare;
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
index e6d056d126d..a8f5b652366 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
@@ -122,7 +122,7 @@ public class HostCapacityMaintainerTest {
public void preprovision_with_shared_host() {
var tester = new DynamicProvisioningTester().addInitialNodes();
// Makes provisioned hosts 48-128-1000-10
- tester.hostProvisioner.overrideHostFlavor("host4");
+ tester.hostProvisioner.setHostFlavor("host4");
var clusterCapacity = new ClusterCapacity(2, 1.0, 30.0, 20.0, 3.0, "fast", "local", "x86_64");
tester.flagSource.withListFlag(PermanentFlags.PREPROVISION_CAPACITY.id(),
List.of(clusterCapacity),
@@ -235,7 +235,7 @@ public class HostCapacityMaintainerTest {
// Pretend shared-host flag has been set to host4's flavor
var sharedHostNodeResources = new NodeResources(48, 128, 1000, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote);
- tester.hostProvisioner.overrideHostFlavor("host4");
+ tester.hostProvisioner.setHostFlavor("host4");
// Next maintenance run does nothing
tester.assertNodesUnchanged();
@@ -365,7 +365,7 @@ public class HostCapacityMaintainerTest {
Cloud cloud = Cloud.builder().dynamicProvisioning(true).build();
DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup());
ProvisioningTester tester = dynamicProvisioningTester.provisioningTester;
- dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default");
+ dynamicProvisioningTester.hostProvisioner.setHostFlavor("default");
// Initial config server hosts are provisioned manually
List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", hostType, 1).stream()
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
index f406f44f02f..6ea5b424d03 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
@@ -256,13 +256,13 @@ public class DynamicProvisioningTest {
ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build();
Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 4, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any)));
- hostProvisioner.overrideHostFlavor("x86");
+ hostProvisioner.setHostFlavor("x86", ClusterSpec.Type.content);
tester.activate(app, cluster, capacity);
NodeList nodes = tester.nodeRepository().nodes().list();
assertEquals(4, nodes.owner(app).state(Node.State.active).size());
assertEquals(Set.of("x86"), nodes.parentsOf(nodes.owner(app).state(Node.State.active)).stream().map(n -> n.flavor().name()).collect(Collectors.toSet()));
- hostProvisioner.overrideHostFlavor("arm");
+ hostProvisioner.setHostFlavor("arm", ClusterSpec.Type.content);
flagSource.withStringFlag(PermanentFlags.HOST_FLAVOR.id(), "arm");
tester.activate(app, cluster, capacity);
nodes = tester.nodeRepository().nodes().list();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
index dcdf79a3951..e59b628ccfd 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
@@ -1,5 +1,5 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.provision.autoscale;
+package com.yahoo.vespa.hosted.provision.provisioning;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
@@ -19,6 +19,10 @@ import com.yahoo.vespa.hosted.provision.Nodelike;
import com.yahoo.vespa.hosted.provision.applications.Application;
import com.yahoo.vespa.hosted.provision.applications.Cluster;
import com.yahoo.vespa.hosted.provision.applications.ScalingEvent;
+import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
+import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling;
+import com.yahoo.vespa.hosted.provision.autoscale.Fixture;
+import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb;
import com.yahoo.vespa.hosted.provision.node.IP;
import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies;
import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
@@ -34,28 +38,43 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
+ * A provisioniong tester which
+ * - Supports dynamic provisioning (only).
+ * - Optionally replicates the actual AWS setup and logic used on Vespa Cloud.
+ * - Supports autoscaling testing.
+ *
+ * TODO: All provisioning testing should migrate to use this, and then the provisionging tester should be collapsed
+ * into this.
+ *
* @author bratseth
*/
-class AutoscalingTester {
+public class DynamicProvisioningTester {
private final ProvisioningTester provisioningTester;
private final Autoscaler autoscaler;
private final HostResourcesCalculator hostResourcesCalculator;
private final CapacityPolicies capacityPolicies;
- public AutoscalingTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) {
+ public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) {
this(zone, hostFlavors, resourcesCalculator, flagSource);
for (Flavor flavor : hostFlavors)
provisioningTester.makeReadyNodes(hostCount, flavor.name(), NodeType.host, 8);
provisioningTester.activateTenantHosts();
}
- private AutoscalingTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) {
+ private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) {
+ MockHostProvisioner hostProvisioner = null;
+ if (zone.cloud().dynamicProvisioning()) {
+ hostProvisioner = new MockHostProvisioner(flavors);
+ hostProvisioner.setHostFlavorIfAvailable(new NodeResources(2, 8, 75, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), resourcesCalculator, ClusterSpec.Type.admin
+ );
+ }
+
provisioningTester = new ProvisioningTester.Builder().zone(zone)
.flavors(flavors)
.resourcesCalculator(resourcesCalculator)
.flagSource(flagSource)
- .hostProvisioner(zone.cloud().dynamicProvisioning() ? new MockHostProvisioner(flavors) : null)
+ .hostProvisioner(hostProvisioner)
.build();
hostResourcesCalculator = resourcesCalculator;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index 68857719bf0..978edf3f7e4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -1041,6 +1041,19 @@ public class ProvisioningTest {
assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources));
}
+ @Test
+ public void testAdminProvisioning() {
+ var nodeResources = new NodeResources(0.25, 1.32, 10, 0.3);
+ var resources = new ClusterResources(1, 1, nodeResources);
+ var fixture = DynamicProvisioningTester.fixture()
+ .awsProdSetup(true)
+ .clusterType(ClusterSpec.Type.admin)
+ .initialResources(Optional.empty())
+ .capacity(Capacity.from(resources))
+ .build();
+ fixture.deploy();
+ }
+
private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size,
int content1Size, NodeResources flavor, ProvisioningTester tester) {
return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42");