Support provisioning of shared hosts

Adds shared-host flag to enable and define resources of shared hosts.  This PR
is a no-op until that flag is set, but there remains some integration with
exclusiveTo (tbd in this PR or follow-up).

6 files changed, 43 insertions, 43 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index 0f70542f141..240e9f49328 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -26,6 +26,7 @@ import com.yahoo.yolean.Exceptions;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -113,18 +114,32 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
 
 
     /**
-     * Provision the nodes necessary to satisfy given capacity.
+     * Provision hosts to ensure there is room to allocate spare nodes.
      *
-     * @return excess hosts that can safely be deprovisioned, if any
+     * @param advertisedSpareCapacity the advertised resources of the spare nodes
+     * @param nodes list of all nodes
+     * @return excess hosts that can safely be deprovisioned: An excess host 1. contains no nodes allocated
+     *         to an application, and assuming the spare nodes have been allocated, and 2. is not parked
+     *         without wantToDeprovision (which means an operator is looking at the node).
      */
-    private List<Node> provision(List<NodeResources> capacity, NodeList nodes) {
-        List<Node> existingHosts = availableHostsOf(nodes);
-        if (nodeRepository().zone().getCloud().dynamicProvisioning()) {
-            existingHosts = removableHostsOf(existingHosts, nodes);
-        } else if (capacity.isEmpty()) {
+    private List<Node> provision(List<NodeResources> advertisedSpareCapacity, NodeList nodes) {
+        if (!nodeRepository().zone().getCloud().dynamicProvisioning()) {
             return List.of();
         }
-        List<Node> excessHosts = new ArrayList<>(existingHosts);
+
+        Map<String, Node> hostsByHostname = new HashMap<>(nodes.hosts().asList().stream()
+                .filter(host -> host.state() != Node.State.parked || host.status().wantToDeprovision())
+                .collect(Collectors.toMap(Node::hostname, Function.identity())));
+
+        nodes.asList().stream()
+                .filter(node -> node.allocation().isPresent())
+                .flatMap(node -> node.parentHostname().stream())
+                .distinct()
+                .forEach(hostsByHostname::remove);
+
+        List<Node> excessHosts = new ArrayList<>(hostsByHostname.values());
+
+        var capacity = new ArrayList<>(advertisedSpareCapacity);
         for (Iterator<NodeResources> it = capacity.iterator(); it.hasNext() && !excessHosts.isEmpty(); ) {
             NodeResources resources = it.next();
             excessHosts.stream()
@@ -138,12 +153,13 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
                            it.remove();
                        });
         }
+
         // Pre-provisioning is best effort, do one host at a time
         capacity.forEach(resources -> {
             try {
                 Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion);
                 List<Node> hosts = hostProvisioner.provisionHosts(nodeRepository().database().getProvisionIndexes(1),
-                                                                  resources, preprovisionAppId, osVersion)
+                                                                  resources, preprovisionAppId, osVersion, false)
                                                   .stream()
                                                   .map(ProvisionedHost::generateHost)
                                                   .collect(Collectors.toList());
@@ -154,7 +170,8 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
                 log.log(Level.WARNING, "Failed to pre-provision " + resources + ", will retry in " + interval(), e);
             }
         });
-        return removableHostsOf(excessHosts, nodes);
+
+        return excessHosts;
     }
 
 
@@ -178,27 +195,4 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer {
             }
         }
     }
-
-    /** Returns hosts that are considered available, i.e. not parked or flagged for deprovisioning */
-    private static List<Node> availableHostsOf(NodeList nodes) {
-        return nodes.hosts()
-                    .matching(host -> host.state() != Node.State.parked || host.status().wantToDeprovision())
-                    .asList();
-    }
-
-    /** Returns the subset of given hosts that have no containers and are thus removable */
-    private static List<Node> removableHostsOf(List<Node> hosts, NodeList allNodes) {
-        Map<String, Node> hostsByHostname = hosts.stream()
-                                                 .collect(Collectors.toMap(Node::hostname,
-                                                                           Function.identity()));
-
-        allNodes.asList().stream()
-                .filter(node -> node.allocation().isPresent())
-                .flatMap(node -> node.parentHostname().stream())
-                .distinct()
-                .forEach(hostsByHostname::remove);
-
-        return List.copyOf(hostsByHostname.values());
-    }
-
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 99557cb0908..f9960716162 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -88,7 +88,7 @@ public class GroupPreparer {
                             .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()),
                                                                                  deficit.getFlavor(),
                                                                                  application,
-                                                                                 osVersion))
+                                                                                 osVersion, false))
                             .orElseGet(List::of);
 
                     // At this point we have started provisioning of the hosts, the first priority is to make sure that
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
index 816ef9459ba..5a7456ab997 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
@@ -21,14 +21,16 @@ public interface HostProvisioner {
      *
      * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames
      *                         on the form of <code>[prefix][index].[domain]</code>
-     * @param resources the resources needed per node
+     * @param resources the resources needed per node - the provisioned host may be significantly larger
      * @param applicationId id of the application that will own the provisioned host
      * @param osVersion the OS version to use. If this version does not exist, implementations may choose a suitable
      *                  fallback version.
+     * @param forceExclusive whether to force the provisioning of an exclusive host.
      * @return list of {@link ProvisionedHost} describing the provisioned nodes
      */
     List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
-                                         ApplicationId applicationId, Version osVersion);
+                                         ApplicationId applicationId, Version osVersion,
+                                         boolean forceExclusive);
 
     /**
      * Continue provisioning of given list of Nodes.
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index 999ceeccc58..03c42bf20ff 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -276,7 +276,9 @@ class AutoscalingTester {
         }
 
         @Override
-        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion) {
+        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
+                                                    ApplicationId applicationId, Version osVersion,
+                                                    boolean forceExclusive) {
             Flavor hostFlavor = hostFlavors.stream().filter(f -> matches(f, resources)).findAny()
                                            .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors));
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
index e75f94123c7..c6052829faf 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
@@ -325,7 +325,8 @@ public class DynamicProvisioningMaintainerTest {
         }
 
         @Override
-        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion) {
+        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources,
+                                                    ApplicationId applicationId, Version osVersion, boolean forceExclusive) {
             Flavor hostFlavor = flavors.getFlavors().stream()
                                        .filter(f -> !f.isDocker())
                                        .filter(f -> f.resources().compatibleWith(resources))
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
index 41f13748a07..c678f8902b5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
@@ -39,6 +39,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -68,7 +69,7 @@ public class DynamicDockerProvisionTest {
 
         mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small"));
         List<HostSpec> hostSpec = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 4, 1, flavor);
-        verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), flavor, application1, Version.emptyVersion);
+        verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), flavor, application1, Version.emptyVersion, false);
 
         // Total of 8 nodes should now be in node-repo, 4 hosts in state provisioned, and 4 reserved nodes
         assertEquals(8, tester.nodeRepository().list().size());
@@ -86,7 +87,7 @@ public class DynamicDockerProvisionTest {
         List<Integer> expectedProvisionIndexes = List.of(100, 101);
         mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("large"));
         tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor);
-        verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor, application, Version.emptyVersion);
+        verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor, application, Version.emptyVersion, false);
 
         // Ready the provisioned hosts, add an IP addresses to pool and activate them
         for (Integer i : expectedProvisionIndexes) {
@@ -101,7 +102,7 @@ public class DynamicDockerProvisionTest {
         mockHostProvisioner(hostProvisioner, tester.nodeRepository().flavors().getFlavorOrThrow("small"));
         tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor);
         // Verify there was only 1 call to provision hosts (during the first prepare)
-        verify(hostProvisioner).provisionHosts(any(), any(), any(), any());
+        verify(hostProvisioner).provisionHosts(any(), any(), any(), any(), anyBoolean());
 
         // Node-repo should now consist of 2 active hosts with 2 reserved nodes on each
         assertEquals(6, tester.nodeRepository().list().size());
@@ -345,7 +346,7 @@ public class DynamicDockerProvisionTest {
             return provisionIndexes.stream()
                     .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor, "host-" + i + "-1", nodeResources, Version.emptyVersion))
                     .collect(Collectors.toList());
-        }).when(hostProvisioner).provisionHosts(any(), any(), any(), any());
+        }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), anyBoolean());
     }
 
     private static class MockHostProvisioner implements HostProvisioner {
@@ -359,7 +360,7 @@ public class DynamicDockerProvisionTest {
         }
 
         @Override
-        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion) {
+        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId, Version osVersion, boolean forceExclusive) {
             Optional<Flavor> hostFlavor = hostFlavors.stream().filter(f -> compatible(f, resources)).findFirst();
             if (hostFlavor.isEmpty())
                 throw new OutOfCapacityException("No host flavor matches " + resources);