Provision additional hosts if needed and dynamic provisioning is enabled

5 files changed, 230 insertions, 4 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 9403729323d..6ff683c1d7e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -75,6 +75,28 @@ public class GroupPreparer {
                 NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes,
                         highestIndex, nodeRepository.zone(), nodeRepository.clock());
                 allocation.offer(prioritizer.prioritize());
+
+                if (dynamicProvisioningEnabled) {
+                    List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit().map(deficit ->
+                            hostProvisioner.get().provisionHosts(deficit.getCount(), deficit.getFlavor())).orElseGet(List::of);
+
+                    // At this point we have started provisioning of the hosts, the first priority is to make sure that
+                    // the returned hosts are added to the node-repo so that they are tracked by the provision maintainers
+                    List<Node> hosts = provisionedHosts.stream()
+                            .map(ProvisionedHost::generateHost)
+                            .collect(Collectors.toList());
+                    nodeRepository.addNodes(hosts);
+
+                    // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit
+                    List<PrioritizableNode> nodes = provisionedHosts.stream()
+                            .map(provisionedHost -> new PrioritizableNode.Builder(provisionedHost.generateNode())
+                                    .withParent(provisionedHost.generateHost())
+                                    .withNewNode(true)
+                                    .build())
+                            .collect(Collectors.toList());
+                    allocation.offer(nodes);
+                }
+
                 if (! allocation.fulfilled() && requestedNodes.canFail())
                     throw new OutOfCapacityException("Could not satisfy " + requestedNodes + " for " + cluster +
                                                      " in " + application.toShortString() +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
index d82e3900e76..34fbaf94478 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
@@ -20,10 +20,9 @@ public interface HostProvisioner {
      * @param numHosts number of hosts to provision
      * @param nodeFlavor Vespa flavor of the node that will run on this host. The resulting provisioned host
      *                   will be of a flavor that is at least as big or bigger than this.
-     * @return list of nodes that should be added to the node-repo, the list will contain exactly 2 elements:
-     * the provisioned host, and a docker container on that host with flavor {@code nodeFlavor}
+     * @return list of {@link ProvisionedHost} describing the provisioned hosts and nodes on them.
      */
-    List<Node> provisionHosts(int numHosts, Flavor nodeFlavor);
+    List<ProvisionedHost> provisionHosts(int numHosts, Flavor nodeFlavor);
 
     /**
      * Continue provisioning of given list of Nodes.
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
new file mode 100644
index 00000000000..eb10a194434
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java
@@ -0,0 +1,69 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Describes a single newly provisioned host by {@link HostProvisioner}.
+ *
+ * @author freva
+ */
+public class ProvisionedHost {
+    private final String id;
+    private final String hostHostname;
+    private final Flavor hostFlavor;
+    private final String nodeHostname;
+    private final Flavor nodeFlavor;
+
+    public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, String nodeHostname, Flavor nodeFlavor) {
+        this.id = Objects.requireNonNull(id, "Host id must be set");
+        this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set");
+        this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set");
+        this.nodeHostname = Objects.requireNonNull(nodeHostname, "Node hostname must be set");
+        this.nodeFlavor = Objects.requireNonNull(nodeFlavor, "Node flavor must be set");
+    }
+
+    /** Generate {@link Node} instance representing the provisioned physical host */
+    Node generateHost() {
+        return Node.create(id, Set.of(), Set.of(), hostHostname, Optional.empty(), hostFlavor, NodeType.host);
+    }
+
+    /** Generate {@link Node} instance representing the node running on this physical host */
+    Node generateNode() {
+        return Node.createDockerNode(Set.of(), Set.of(), nodeHostname, hostHostname, nodeFlavor, NodeType.tenant);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ProvisionedHost that = (ProvisionedHost) o;
+        return id.equals(that.id) &&
+                hostHostname.equals(that.hostHostname) &&
+                hostFlavor.equals(that.hostFlavor) &&
+                nodeHostname.equals(that.nodeHostname) &&
+                nodeFlavor.equals(that.nodeFlavor);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, hostHostname, hostFlavor, nodeHostname, nodeFlavor);
+    }
+
+    @Override
+    public String toString() {
+        return "ProvisionedHost{" +
+                "id='" + id + '\'' +
+                ", hostHostname='" + hostHostname + '\'' +
+                ", hostFlavor=" + hostFlavor +
+                ", nodeHostname='" + nodeHostname + '\'' +
+                ", nodeFlavor=" + nodeFlavor +
+                '}';
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
new file mode 100644
index 00000000000..1d1fd3d1b99
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
@@ -0,0 +1,136 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author freva
+ */
+public class DynamicDockerProvisionTest {
+
+    private final MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup();
+    private final HostProvisioner hostProvisioner = mock(HostProvisioner.class);
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource()
+            .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true);
+    private final ProvisioningTester tester = new ProvisioningTester.Builder()
+            .hostProvisioner(hostProvisioner).flagSource(flagSource).nameResolver(nameResolver).build();
+
+    @Test
+    public void dynamically_provision_with_empty_node_repo() {
+        assertEquals(0, tester.nodeRepository().list().size());
+
+        ApplicationId application1 = tester.makeApplicationId();
+        Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("dockerSmall");
+
+        mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small"));
+        List<HostSpec> hostSpec = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 4, 1, flavor.canonicalName());
+        verify(hostProvisioner).provisionHosts(4, flavor);
+
+        // Total of 8 nodes should now be in node-repo, 4 hosts in state provisioned, and 4 reserved nodes
+        assertEquals(8, tester.nodeRepository().list().size());
+        assertEquals(4, tester.nodeRepository().getNodes(NodeType.host, Node.State.provisioned).size());
+        assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size());
+        assertEquals(List.of("host-1-1", "host-2-1", "host-3-1", "host-4-1"),
+                hostSpec.stream().map(HostSpec::hostname).collect(Collectors.toList()));
+    }
+
+    @Test
+    public void does_not_allocate_to_available_empty_hosts() {
+        tester.makeReadyNodes(3, "small", NodeType.host, 10);
+        deployZoneApp(tester);
+
+        ApplicationId application = tester.makeApplicationId();
+        Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("dockerSmall");
+
+        mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small"));
+        tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor.canonicalName());
+        verify(hostProvisioner).provisionHosts(2, flavor);
+    }
+
+    @Test
+    public void allocates_to_hosts_already_hosting_nodes_by_this_tenant() {
+        ApplicationId application = tester.makeApplicationId();
+        Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("dockerSmall");
+
+        mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("large"));
+        tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor.canonicalName());
+        verify(hostProvisioner).provisionHosts(2, flavor);
+
+        // Ready the provisioned hosts, add an IP addreses to pool and activate them
+        for (int i = 1; i < 3; i++) {
+            String hostname = "host-" + i;
+            Node host = tester.nodeRepository().getNode(hostname).orElseThrow()
+                    .withIpAddressPool(Set.of("::" + i + ":2")).withIpAddresses(Set.of("::" + i + ":0"));
+            tester.nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName());
+            nameResolver.addRecord(hostname + "-2", "::" + i + ":2");
+        }
+        deployZoneApp(tester);
+
+        mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small"));
+        tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor.canonicalName());
+        // Verify there was only 1 call to provision hosts (during the first prepare)
+        verify(hostProvisioner).provisionHosts(anyInt(), any());
+
+        // Node-repo should now consist of 2 active hosts with 2 reserved nodes on each
+        assertEquals(6, tester.nodeRepository().list().size());
+        assertEquals(2, tester.nodeRepository().getNodes(NodeType.host, Node.State.active).size());
+        assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size());
+    }
+
+
+    private static void deployZoneApp(ProvisioningTester tester) {
+        ApplicationId applicationId = tester.makeApplicationId();
+        List<HostSpec> list = tester.prepare(applicationId,
+                ClusterSpec.request(ClusterSpec.Type.container,
+                        ClusterSpec.Id.from("node-admin"),
+                        Version.fromString("6.42"),
+                        false, Collections.emptySet()),
+                Capacity.fromRequiredNodeType(NodeType.host),
+                1);
+        tester.activate(applicationId, ImmutableSet.copyOf(list));
+    }
+
+
+    private static ClusterSpec clusterSpec(String clusterId) {
+        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false, Collections.emptySet());
+    }
+
+    private static void mockHostProvisioner(HostProvisioner hostProvisioner, Flavor hostFlavor) {
+        final int[] numProvisioned = { 0 };
+        doAnswer(invocation -> {
+            int numHosts = (int) invocation.getArguments()[0];
+            Flavor nodeFlavor = (Flavor) invocation.getArguments()[1];
+            System.out.println(numHosts + " " + nodeFlavor);
+            return IntStream.range(0, numHosts)
+                    .map(i -> ++numProvisioned[0])
+                    .mapToObj(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor, "host-" + i + "-1", nodeFlavor))
+                    .collect(Collectors.toList());
+        }).when(hostProvisioner).provisionHosts(anyInt(), any());
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 7109a81539c..eeebe4de256 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -452,7 +452,7 @@ public class ProvisioningTester {
                     Optional.ofNullable(zone).orElseGet(Zone::defaultZone),
                     Optional.ofNullable(nameResolver).orElseGet(() -> new MockNameResolver().mockAnyLookup()),
                     orchestrator,
-                    Optional.ofNullable(hostProvisioner).orElseGet(() -> null),
+                    Optional.ofNullable(hostProvisioner).orElse(null),
                     Optional.ofNullable(loadBalancerService).orElseGet(LoadBalancerServiceMock::new),
                     Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new));
         }