Tune cluster controller memory

- Change from 7 to 6 cluster controllers per host
- Correct real memory limit for admin
- Test admin provisioning

9 files changed, 76 insertions, 21 deletions

diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
index b7a3589a4d0..4c4d9c78c8e 100644
--- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -775,7 +775,7 @@ public class DocumentSelectorTestCase {
 
     @Test
     public void testInheritance() throws ParseException {
-        var s=new DocumentSelector("parent.parentField = \"parentValue\"");
+        new DocumentSelector("parent.parentField = \"parentValue\"");
         List<DocumentPut> documents = createDocs();
         assertEquals(Result.TRUE, evaluate("test", documents.get(0)));
         assertEquals("Matching on type is exact", Result.FALSE, evaluate("parent", documents.get(0)));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 8af1df93bde..5732e94956a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -115,10 +115,12 @@ public class CapacityPolicies {
         if (nodeRepository.exclusiveAllocation(clusterSpec)) {
             return versioned(clusterSpec, Map.of(new Version(0), smallestExclusiveResources()));
         }
-        // TODO (hmusum): Go back to 1.14 Gb memory when bug in resource limits for admin nodes
-        // has been fixed
+
+        // 1.32 fits floor(8/1.32) = 6 cluster controllers on each 8Gb host, and each will have
+        // 1.32-(0.7+0.6)*(1.32/8) = 1.1 Gb real memory given current taxes.
         return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.14, 10, 0.3),
-                                             new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3)));
+                                             new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3),
+                                             new Version(8, 129,  4), new NodeResources(0.25, 1.32, 10, 0.3)));
     }
 
     private Architecture adminClusterArchitecture(ApplicationId instance) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
index 1acb69113c1..4d33e1c7bad 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
@@ -88,10 +88,13 @@ public class NodeResourceLimits {
             return 4;
     }
 
-    private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) { return minAdvertisedVcpu(applicationId, cluster); }
+    private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) {
+        return minAdvertisedVcpu(applicationId, cluster);
+    }
 
     private double minRealMemoryGb(ClusterSpec cluster) {
-        return minAdvertisedMemoryGb(cluster) - 1.7;
+        if (cluster.type() == ClusterSpec.Type.admin) return 0.95; // TODO: Increase to 1.05 after March 2023
+        return 2.3;
     }
 
     private double minRealDiskGb() { return 6; }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
index 84856ab310b..ad53dc8727d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
 
 import java.time.Instant;
@@ -46,7 +47,7 @@ public class MockHostProvisioner implements HostProvisioner {
 
     private int deprovisionedHosts = 0;
     private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class);
-    private Optional<Flavor> hostFlavor = Optional.empty();
+    private Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>();
 
     public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb) {
         this.flavors = List.copyOf(flavors);
@@ -67,11 +68,14 @@ public class MockHostProvisioner implements HostProvisioner {
                                ApplicationId applicationId, Version osVersion, HostSharing sharing,
                                Optional<ClusterSpec.Type> clusterType, CloudAccount cloudAccount,
                                Consumer<List<ProvisionedHost>> provisionedHostsConsumer) {
-        Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream()
-                                                                   .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources)
-                                                                                                                 : f.resources().satisfies(resources))
-                                                                   .findFirst()
-                                                                   .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true)));
+        Flavor hostFlavor = hostFlavors.get(clusterType.orElse(ClusterSpec.Type.content));
+        if (hostFlavor == null)
+            hostFlavor = flavors.stream()
+                                .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources)
+                                                                              : f.resources().satisfies(resources))
+                                .findFirst()
+                                .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true));
+
         List<ProvisionedHost> hosts = new ArrayList<>();
         for (int index : provisionIndices) {
             String hostHostname = hostType == NodeType.host ? "host" + index : hostType.name() + index;
@@ -152,14 +156,30 @@ public class MockHostProvisioner implements HostProvisioner {
         return this;
     }
 
-    public MockHostProvisioner overrideHostFlavor(String flavorName) {
+    public MockHostProvisioner setHostFlavor(String flavorName, ClusterSpec.Type ... types) {
         Flavor flavor = flavors.stream().filter(f -> f.name().equals(flavorName))
                                .findFirst()
                                .orElseThrow(() -> new IllegalArgumentException("No such flavor '" + flavorName + "'"));
-        hostFlavor = Optional.of(flavor);
+        if (types.length == 0)
+            types = ClusterSpec.Type.values();
+        for (var type : types)
+            hostFlavors.put(type, flavor);
         return this;
     }
 
+    /** Sets the host flavor to use to the flavor matching these resources exactly, if any. */
+    public MockHostProvisioner setHostFlavorIfAvailable(NodeResources flavorAdvertisedResources, HostResourcesCalculator calculator, ClusterSpec.Type ... types) {
+        Optional<Flavor> hostFlavor = flavors.stream().filter(f -> calculator.advertisedResourcesOf(f).compatibleWith(flavorAdvertisedResources))
+                                             .findFirst();
+        if (types.length == 0)
+            types = ClusterSpec.Type.values();
+        for (var type : types)
+            hostFlavor.ifPresent(f -> hostFlavors.put(type, f));
+        return this;
+    }
+
+    public Optional<Flavor> getHostFlavor(ClusterSpec.Type type) { return Optional.ofNullable(hostFlavors.get(type)); }
+
     public MockHostProvisioner addEvent(HostEvent event) {
         hostEvents.add(event);
         return this;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index dcdf79a3951..a275bf4435a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -34,9 +34,17 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 /**
+ * A provisioniong tester which
+ * - Supports dynamic provisioning (only).
+ * - Optionally replicates the actual AWS setup and logic used on Vespa Cloud.
+ * - Supports autoscaling testing.
+ *
+ * TODO: All provisioning testing should migrate to use this, and then the provisionging tester should be collapsed
+ *       into this.
+ *
  * @author bratseth
  */
-class AutoscalingTester {
+public class AutoscalingTester {
 
     private final ProvisioningTester provisioningTester;
     private final Autoscaler autoscaler;
@@ -51,11 +59,18 @@ class AutoscalingTester {
     }
 
     private AutoscalingTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) {
+        MockHostProvisioner hostProvisioner = null;
+        if (zone.cloud().dynamicProvisioning()) {
+            hostProvisioner = new MockHostProvisioner(flavors);
+            hostProvisioner.setHostFlavorIfAvailable(new NodeResources(2, 8, 75, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), resourcesCalculator, ClusterSpec.Type.admin
+            );
+        }
+
         provisioningTester = new ProvisioningTester.Builder().zone(zone)
                                                              .flavors(flavors)
                                                              .resourcesCalculator(resourcesCalculator)
                                                              .flagSource(flagSource)
-                                                             .hostProvisioner(zone.cloud().dynamicProvisioning() ? new MockHostProvisioner(flavors) : null)
+                                                             .hostProvisioner(hostProvisioner)
                                                              .build();
 
         hostResourcesCalculator = resourcesCalculator;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java
index 96fa143dc57..69469bb03c7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java
@@ -48,6 +48,7 @@ public class AwsResourcesCalculator {
                              ( hostFlavor.advertisedResources().memoryGb() - ( real ? hostMemoryOverhead : 0));
         if (memoryShare > 1) // The real resources of the host cannot fit the requested real resources after overhead
             memoryShare = 1;
+
         return hostMemoryOverhead * memoryShare;
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
index e6d056d126d..a8f5b652366 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
@@ -122,7 +122,7 @@ public class HostCapacityMaintainerTest {
     public void preprovision_with_shared_host() {
         var tester = new DynamicProvisioningTester().addInitialNodes();
         // Makes provisioned hosts 48-128-1000-10
-        tester.hostProvisioner.overrideHostFlavor("host4");
+        tester.hostProvisioner.setHostFlavor("host4");
         var clusterCapacity = new ClusterCapacity(2, 1.0, 30.0, 20.0, 3.0, "fast", "local", "x86_64");
         tester.flagSource.withListFlag(PermanentFlags.PREPROVISION_CAPACITY.id(),
                 List.of(clusterCapacity),
@@ -235,7 +235,7 @@ public class HostCapacityMaintainerTest {
 
         // Pretend shared-host flag has been set to host4's flavor
         var sharedHostNodeResources = new NodeResources(48, 128, 1000, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote);
-        tester.hostProvisioner.overrideHostFlavor("host4");
+        tester.hostProvisioner.setHostFlavor("host4");
 
         // Next maintenance run does nothing
         tester.assertNodesUnchanged();
@@ -365,7 +365,7 @@ public class HostCapacityMaintainerTest {
         Cloud cloud = Cloud.builder().dynamicProvisioning(true).build();
         DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup());
         ProvisioningTester tester = dynamicProvisioningTester.provisioningTester;
-        dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default");
+        dynamicProvisioningTester.hostProvisioner.setHostFlavor("default");
 
         // Initial config server hosts are provisioned manually
         List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", hostType, 1).stream()
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
index f406f44f02f..6ea5b424d03 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java
@@ -256,13 +256,13 @@ public class DynamicProvisioningTest {
         ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build();
         Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 4, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any)));
 
-        hostProvisioner.overrideHostFlavor("x86");
+        hostProvisioner.setHostFlavor("x86", ClusterSpec.Type.content);
         tester.activate(app, cluster, capacity);
         NodeList nodes = tester.nodeRepository().nodes().list();
         assertEquals(4, nodes.owner(app).state(Node.State.active).size());
         assertEquals(Set.of("x86"), nodes.parentsOf(nodes.owner(app).state(Node.State.active)).stream().map(n -> n.flavor().name()).collect(Collectors.toSet()));
 
-        hostProvisioner.overrideHostFlavor("arm");
+        hostProvisioner.setHostFlavor("arm", ClusterSpec.Type.content);
         flagSource.withStringFlag(PermanentFlags.HOST_FLAVOR.id(), "arm");
         tester.activate(app, cluster, capacity);
         nodes = tester.nodeRepository().nodes().list();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index 68857719bf0..f163e4d33b9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -24,6 +24,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.autoscale.AutoscalingTester;
 import com.yahoo.vespa.hosted.provision.maintenance.ReservationExpirer;
 import com.yahoo.vespa.hosted.provision.maintenance.TestMetric;
 import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -1041,6 +1042,19 @@ public class ProvisioningTest {
         assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources));
     }
 
+    @Test
+    public void testAdminProvisioning() {
+        var nodeResources = new NodeResources(0.25, 1.32, 10, 0.3);
+        var resources = new ClusterResources(1, 1, nodeResources);
+        var fixture = AutoscalingTester.fixture()
+                                       .awsProdSetup(true)
+                                       .clusterType(ClusterSpec.Type.admin)
+                                       .initialResources(Optional.empty())
+                                       .capacity(Capacity.from(resources))
+                                       .build();
+        fixture.deploy();
+    }
+
     private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size,
                                 int content1Size, NodeResources flavor, ProvisioningTester tester) {
         return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42");