Test reservation of hosts to tenants

5 files changed, 107 insertions, 43 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 9279c4e2332..d5aaf5dbad7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.config.provisioning.NodeRepositoryConfig;
 import com.yahoo.transaction.Mutex;
@@ -309,16 +310,15 @@ public class NodeRepository extends AbstractComponent {
 
     /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */
     public Node createNode(String openStackId, String hostname, IP.Config ipConfig, Optional<String> parentHostname,
-                           Flavor flavor, NodeType type) {
+                           Flavor flavor, Optional<TenantName> reservedTo, NodeType type) {
         if (ipConfig.primary().isEmpty()) { // TODO: Remove this. Only test code hits this path
             ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname));
         }
-        return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, Optional.empty(), type);
+        return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type);
     }
 
-    public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor,
-                           NodeType type) {
-        return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, type);
+    public Node createNode(String openStackId, String hostname, Optional<String> parentHostname, Flavor flavor, NodeType type) {
+        return createNode(openStackId, hostname, IP.Config.EMPTY, parentHostname, flavor, Optional.empty(), type);
     }
 
     /** Adds a list of newly created docker container nodes to the node repository as <i>reserved</i> nodes */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 0d331ac0f3f..1817470a63b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -70,34 +70,34 @@ public class MockNodeRepository extends NodeRepository {
 
         // Regular nodes
         nodes.add(createNode("node1", "host1.yahoo.com", ipConfig(1), Optional.empty(),
-                             new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant));
+                             new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant));
         nodes.add(createNode("node2", "host2.yahoo.com", ipConfig(2), Optional.empty(),
-                             new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant));
+                             new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant));
         nodes.add(createNode("node3", "host3.yahoo.com", ipConfig(3), Optional.empty(),
-                             new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), NodeType.tenant));
+                             new Flavor(new NodeResources(0.5, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
         Node node4 = createNode("node4", "host4.yahoo.com", ipConfig(4), Optional.of("dockerhost1.yahoo.com"),
-                                new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), NodeType.tenant);
+                                new Flavor(new NodeResources(1, 4, 100, 1, fast, local)), Optional.empty(), NodeType.tenant);
         node4 = node4.with(node4.status()
                                 .withVespaVersion(new Version("6.41.0"))
                                 .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:6.41.0")));
         nodes.add(node4);
 
         Node node5 = createNode("node5", "host5.yahoo.com", ipConfig(5), Optional.of("dockerhost2.yahoo.com"),
-                                new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), NodeType.tenant);
+                                new Flavor(new NodeResources(1, 8, 100, 1, slow, remote)), Optional.empty(), NodeType.tenant);
         nodes.add(node5.with(node5.status()
                                   .withVespaVersion(new Version("1.2.3"))
                                   .withDockerImage(DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa:1.2.3"))));
 
 
         nodes.add(createNode("node6", "host6.yahoo.com", ipConfig(6), Optional.empty(),
-                             new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant));
+                             new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant));
         Node node7 = createNode("node7", "host7.yahoo.com", ipConfig(7), Optional.empty(),
-                                new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant);
+                                new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant);
         nodes.add(node7);
 
         // 8, 9, 11 and 12 are added by web service calls
         Node node10 = createNode("node10", "host10.yahoo.com", ipConfig(10), Optional.of("parent1.yahoo.com"),
-                                 new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant);
+                                 new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant);
         Status node10newStatus = node10.status();
         node10newStatus = node10newStatus
                 .withVespaVersion(Version.fromString("5.104.142"))
@@ -106,26 +106,26 @@ public class MockNodeRepository extends NodeRepository {
         nodes.add(node10);
 
         Node node55 = createNode("node55", "host55.yahoo.com", ipConfig(55), Optional.empty(),
-                                 new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), NodeType.tenant);
+                                 new Flavor(new NodeResources(2, 8, 50, 1, fast, local)), Optional.empty(), NodeType.tenant);
         nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true)));
 
         /* Setup docker hosts (two of these will be reserved for spares */
         nodes.add(createNode("dockerhost1", "dockerhost1.yahoo.com", ipConfig(100, 1, 3), Optional.empty(),
-                             flavors.getFlavorOrThrow("large"), NodeType.host));
+                             flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
         nodes.add(createNode("dockerhost2", "dockerhost2.yahoo.com", ipConfig(101, 1, 3), Optional.empty(),
-                             flavors.getFlavorOrThrow("large"), NodeType.host));
+                             flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
         nodes.add(createNode("dockerhost3", "dockerhost3.yahoo.com", ipConfig(102, 1, 3), Optional.empty(),
-                             flavors.getFlavorOrThrow("large"), NodeType.host));
+                             flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
         nodes.add(createNode("dockerhost4", "dockerhost4.yahoo.com", ipConfig(103, 1, 3), Optional.empty(),
-                             flavors.getFlavorOrThrow("large"), NodeType.host));
+                             flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
         nodes.add(createNode("dockerhost5", "dockerhost5.yahoo.com", ipConfig(104, 1, 3), Optional.empty(),
-                             flavors.getFlavorOrThrow("large"), NodeType.host));
+                             flavors.getFlavorOrThrow("large"), Optional.empty(), NodeType.host));
 
         // Config servers
         nodes.add(createNode("cfg1", "cfg1.yahoo.com", ipConfig(201), Optional.empty(),
-                             flavors.getFlavorOrThrow("default"), NodeType.config));
+                             flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config));
         nodes.add(createNode("cfg2", "cfg2.yahoo.com", ipConfig(202), Optional.empty(),
-                             flavors.getFlavorOrThrow("default"), NodeType.config));
+                             flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config));
 
         // Ready all nodes, except 7 and 55
         nodes = addNodes(nodes);
@@ -167,9 +167,9 @@ public class MockNodeRepository extends NodeRepository {
 
         List<Node> largeNodes = new ArrayList<>();
         largeNodes.add(createNode("node13", "host13.yahoo.com", ipConfig(13), Optional.empty(),
-                                  new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant));
+                                  new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
         largeNodes.add(createNode("node14", "host14.yahoo.com", ipConfig(14), Optional.empty(),
-                                  new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), NodeType.tenant));
+                                  new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
         addNodes(largeNodes);
         setReady(largeNodes, Agent.system, getClass().getSimpleName());
         ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
index da174a8d38c..afac44856c9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
@@ -133,8 +133,8 @@ public class CapacityCheckerTester {
             NodeResources nr = containingNodeResources(childResources,
                     excessCapacity);
             Node node = nodeRepository.createNode(hostname, hostname,
-                    new IP.Config(Set.of("::"), availableIps), Optional.empty(),
-                    new Flavor(nr), NodeType.host);
+                                                  new IP.Config(Set.of("::"), availableIps), Optional.empty(),
+                                                  new Flavor(nr), Optional.empty(), NodeType.host);
             hosts.add(node);
         }
         return hosts;
@@ -152,8 +152,8 @@ public class CapacityCheckerTester {
                     .mapToObj(n -> String.format("%04X::%04X", hostid, n))
                     .collect(Collectors.toSet());
             Node node = nodeRepository.createNode(hostname, hostname,
-                    new IP.Config(Set.of("::"), availableIps), Optional.empty(),
-                    new Flavor(capacity), NodeType.host);
+                                                  new IP.Config(Set.of("::"), availableIps), Optional.empty(),
+                                                  new Flavor(capacity), Optional.empty(), NodeType.host);
             hosts.add(node);
         }
         return hosts;
@@ -263,8 +263,8 @@ public class CapacityCheckerTester {
         Flavor f = new Flavor(nr);
 
         Node node = nodeRepository.createNode(nodeModel.id, nodeModel.hostname,
-                new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses),
-                nodeModel.parentHostname, f, nodeModel.type);
+                                              new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses),
+                                              nodeModel.parentHostname, f, Optional.empty(), nodeModel.type);
 
         if (membership != null) {
             return node.allocate(owner, membership, node.flavor().resources(), Instant.now());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
index b731ab309a6..d269b739feb 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
@@ -14,6 +14,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.OutOfCapacityException;
 import com.yahoo.config.provision.ParentHostUnavailableException;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -107,6 +108,49 @@ public class DockerProvisioningTest {
         assertEquals(nodeCount, activeNodes.size());
     }
 
+    @Test
+    public void reservations_are_respected() {
+        NodeResources resources = new NodeResources(10, 10, 10, 10);
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+        TenantName tenant1 = TenantName.from("tenant1");
+        TenantName tenant2 = TenantName.from("tenant2");
+        ApplicationId application1_1 = ApplicationId.from(tenant1, ApplicationName.from("application1"), InstanceName.defaultName());
+        ApplicationId application2_1 = ApplicationId.from(tenant2, ApplicationName.from("application1"), InstanceName.defaultName());
+        ApplicationId application2_2 = ApplicationId.from(tenant2, ApplicationName.from("application2"), InstanceName.defaultName());
+
+        List<Node> tenant1Parents = tester.makeReadyNodes(10, resources, Optional.of(tenant1), NodeType.host, 1);
+        List<Node> nonreservedParents = tester.makeReadyNodes(10, resources, Optional.empty(), NodeType.host, 1);
+        tester.deployZoneApp();
+
+        Version wantedVespaVersion = Version.fromString("6.39");
+        List<HostSpec> nodes = tester.prepare(application2_1,
+                                              ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                                              6, 1, resources);
+        assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1
+        tester.activate(application2_1, nodes);
+        System.out.println("Allocated to application 2_1:\n-----------------------------------");
+        tester.nodeRepository().getNodes(application2_1).forEach(n -> System.out.println(n.hostname() + " reservedTo " + tester.nodeRepository().getNode(n.hostname()).get().reservedTo()));
+
+        try {
+            tester.prepare(application2_2,
+                           ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                           5, 1, resources);
+            System.out.println("Allocated to application 2_2:\n-----------------------------------");
+            tester.nodeRepository().getNodes(application2_2).forEach(n -> System.out.println(n.hostname() + " reservedTo " + tester.nodeRepository().getNode(n.hostname()).get().reservedTo()));
+            fail("Expected exception");
+        }
+        catch (OutOfCapacityException e) {
+            // Success: Not enough nonreserved hosts left
+        }
+
+        nodes = tester.prepare(application1_1,
+                               ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                              10, 1, resources);
+        assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester);
+        tester.activate(application1_1, nodes);
+        assertNodeParentReservation(tester.getNodes(application1_1).asList(), Optional.empty(), tester); // Reservation is cleared after activation
+    }
+
     /** Exclusive app first, then non-exclusive: Should give the same result as below */
     @Test
     public void docker_application_deployment_with_exclusive_app_first() {
@@ -260,4 +304,16 @@ public class DockerProvisioningTest {
         tester.activate(application, hosts);
     }
 
+    private void assertNodeParentReservation(List<Node> nodes, Optional<TenantName> reservation, ProvisioningTester tester) {
+        for (Node node : nodes)
+            assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo());
+    }
+
+    private void assertHostSpecParentReservation(List<HostSpec> hostSpecs, Optional<TenantName> reservation, ProvisioningTester tester) {
+        for (HostSpec hostSpec : hostSpecs) {
+            Node node = tester.nodeRepository().getNode(hostSpec.hostname()).get();
+            assertEquals(reservation, tester.nodeRepository().getNode(node.parentHostname().get()).get().reservedTo());
+        }
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 8be0da7c83d..4e63d3cc79f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -241,13 +241,16 @@ public class ProvisioningTester {
     }
 
     public List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
-        return makeReadyNodes(n, asFlavor(flavor, type), type, 0);
+        return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, 0);
     }
     public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type) {
-        return makeReadyNodes(n, new Flavor(resources), type, 0);
+        return makeReadyNodes(n, new Flavor(resources), Optional.empty(), type, 0);
     }
     public List<Node> makeReadyNodes(int n, NodeResources resources, NodeType type, int ipAddressPoolSize) {
-        return makeReadyNodes(n, new Flavor(resources), type, ipAddressPoolSize);
+        return makeReadyNodes(n, resources, Optional.empty(), type, ipAddressPoolSize);
+    }
+    public List<Node> makeReadyNodes(int n, NodeResources resources, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) {
+        return makeReadyNodes(n, new Flavor(resources), reservedTo, type, ipAddressPoolSize);
     }
 
     public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) {
@@ -255,9 +258,9 @@ public class ProvisioningTester {
     }
 
     public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
-        return makeProvisionedNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack);
+        return makeProvisionedNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize, dualStack);
     }
-    public List<Node> makeProvisionedNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+    public List<Node> makeProvisionedNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         List<Node> nodes = new ArrayList<>(n);
 
         for (int i = 0; i < n; i++) {
@@ -299,6 +302,7 @@ public class ProvisioningTester {
                                                 new IP.Config(hostIps, ipAddressPool),
                                                 Optional.empty(),
                                                 flavor,
+                                                reservedTo,
                                                 type));
         }
         nodes = nodeRepository.addNodes(nodes);
@@ -315,11 +319,12 @@ public class ProvisioningTester {
 
             nameResolver.addRecord(hostname, ipv4);
             Node node = nodeRepository.createNode(hostname,
-                    hostname,
-                    new IP.Config(Set.of(ipv4), Set.of()),
-                    Optional.empty(),
-                    nodeFlavors.getFlavorOrThrow(flavor),
-                    NodeType.config);
+                                                  hostname,
+                                                  new IP.Config(Set.of(ipv4), Set.of()),
+                                                  Optional.empty(),
+                                                  nodeFlavors.getFlavorOrThrow(flavor),
+                                                  Optional.empty(),
+                                                  NodeType.config);
             nodes.add(node);
         }
 
@@ -338,17 +343,20 @@ public class ProvisioningTester {
     }
 
     public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) {
-        return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize);
+        return makeReadyNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize);
     }
-    public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize) {
-        return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false);
+    public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize) {
+        return makeReadyNodes(n, flavor, reservedTo, type, ipAddressPoolSize, false);
     }
 
     public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         return makeReadyNodes(n, asFlavor(flavor, type), type, ipAddressPoolSize, dualStack);
     }
     public List<Node> makeReadyNodes(int n, Flavor flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
-        List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack);
+        return makeReadyNodes(n, flavor, Optional.empty(), type, ipAddressPoolSize, dualStack);
+    }
+    public List<Node> makeReadyNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+        List<Node> nodes = makeProvisionedNodes(n, flavor, reservedTo, type, ipAddressPoolSize, dualStack);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }