Add InfrastructureProvisioner

5 files changed, 262 insertions, 3 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
new file mode 100644
index 00000000000..c958be13a5d
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -0,0 +1,99 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.log.LogLevel;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
+import com.yahoo.vespa.service.monitor.application.ConfigServerHostApplication;
+import com.yahoo.vespa.service.monitor.application.HostedVespaApplication;
+import com.yahoo.vespa.service.monitor.application.ProxyHostApplication;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * @author freva
+ */
+public class InfrastructureProvisioner extends Maintainer {
+
+    private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName());
+    private static final List<HostedVespaApplication> HOSTED_VESPA_APPLICATIONS = Arrays.asList(
+            ConfigServerApplication.CONFIG_SERVER_APPLICATION,
+            ConfigServerHostApplication.CONFIG_SERVER_HOST_APPLICATION,
+            ProxyHostApplication.PROXY_HOST_APPLICATION);
+
+    private final Provisioner provisioner;
+    private final InfrastructureVersions infrastructureVersions;
+
+    public InfrastructureProvisioner(Provisioner provisioner, NodeRepository nodeRepository,
+                                     InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl) {
+        super(nodeRepository, interval, jobControl);
+        this.provisioner = provisioner;
+        this.infrastructureVersions = infrastructureVersions;
+    }
+
+    @Override
+    protected void maintain() {
+        for (HostedVespaApplication application: HOSTED_VESPA_APPLICATIONS) {
+            try (Mutex lock = nodeRepository().lock(application.getApplicationId())) {
+                Optional<Version> version = getVersionToProvision(application.getCapacity().type());
+                if (! version.isPresent()) continue;
+
+                List<HostSpec> hostSpecs = provisioner.prepare(
+                        application.getApplicationId(),
+                        application.getClusterSpecWithVersion(version.get()),
+                        application.getCapacity(),
+                        1, // groups
+                        logger::log);
+
+                NestedTransaction nestedTransaction = new NestedTransaction();
+                provisioner.activate(nestedTransaction, application.getApplicationId(), hostSpecs);
+                nestedTransaction.commit();
+            }
+        }
+    }
+
+    /**
+     * Returns the version that the given node type should be provisioned to. This is
+     * the version returned by {@link InfrastructureVersions#getTargetVersionFor} unless a provisioning is:
+     * <ul>
+     *   <li>not possible: no nodes of given type in legal state in node-repo</li>
+     *   <li>redudant: all nodes that can be provisioned already have the right wanted Vespa version</li>
+     * </ul>
+     */
+    Optional<Version> getVersionToProvision(NodeType nodeType) {
+        Optional<Version> wantedWantedVersion = infrastructureVersions.getTargetVersionFor(nodeType);
+        if (!wantedWantedVersion.isPresent()) {
+            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set");
+            return Optional.empty();
+        }
+
+        List<Optional<Version>> currentWantedVersions = nodeRepository().getNodes(nodeType,
+                Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive).stream()
+                .map(node -> node.allocation()
+                        .map(allocation -> allocation.membership().cluster().vespaVersion()))
+                .collect(Collectors.toList());
+        if (currentWantedVersions.isEmpty()) {
+            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision");
+            return Optional.empty();
+        }
+
+        if (currentWantedVersions.stream().allMatch(wantedWantedVersion::equals)) {
+            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType +
+                    ": Already provisioned to wanted version " + wantedWantedVersion);
+            return Optional.empty();
+        }
+        return wantedWantedVersion;
+    }
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 7e8cd65d37f..32c7a4035d9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -7,6 +7,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostLivenessTracker;
+import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -46,20 +47,21 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final NodeRebooter nodeRebooter;
     private final NodeRetirer nodeRetirer;
     private final MetricsReporter metricsReporter;
+    private final InfrastructureProvisioner infrastructureProvisioner;
 
     private final JobControl jobControl;
     private final InfrastructureVersions infrastructureVersions;
 
     @Inject
-    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer,
+    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig) {
-        this(nodeRepository, deployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
+        this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
                 orchestrator, metric, configserverConfig);
     }
 
-    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer,
+    public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig) {
@@ -78,6 +80,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl);
         nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl);
         metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl);
+        infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl);
+
 
         RetirementPolicy policy = new RetirementPolicyList(new RetireIPv4OnlyNodes(zone));
         FlavorSpareChecker flavorSpareChecker = new FlavorSpareChecker(
@@ -99,6 +103,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         nodeRetirer.deconstruct();
         provisionedExpirer.deconstruct();
         metricsReporter.deconstruct();
+        infrastructureProvisioner.deconstruct();
     }
 
     public JobControl jobControl() { return jobControl; }
@@ -142,6 +147,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration nodeRetirerInterval;
         private final Duration metricsInterval;
         private final Duration retiredInterval;
+        private final Duration infrastructureProvisionInterval;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -154,6 +160,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             rebootInterval = Duration.ofDays(30);
             nodeRetirerInterval = Duration.ofMinutes(30);
             metricsInterval = Duration.ofMinutes(1);
+            infrastructureProvisionInterval = Duration.ofMinutes(3);
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
 
             if (environment.isTest())
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index c5067c0f959..6c43ed18645 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -16,6 +16,7 @@ public class ContainerConfig {
                         "  <component id='com.yahoo.vespa.curator.mock.MockCurator'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDeployer'/>" +
+                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>" +
                         "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
new file mode 100644
index 00000000000..44ee9390e65
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
@@ -0,0 +1,42 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.google.inject.Inject;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostFilter;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ProvisionLogger;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.transaction.NestedTransaction;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+public class MockProvisioner implements Provisioner {
+
+    @Inject
+    public MockProvisioner() {}
+
+    @Override
+    public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
+
+    }
+
+    @Override
+    public void remove(NestedTransaction transaction, ApplicationId application) {
+
+    }
+
+    @Override
+    public void restart(ApplicationId application, HostFilter filter) {
+
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
new file mode 100644
index 00000000000..56c7114064b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
@@ -0,0 +1,110 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.Provisioner;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Generation;
+import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
+
+import java.time.Duration;
+import java.util.Optional;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author freva
+ */
+public class InfrastructureProvisionerTest {
+
+    private final NodeRepositoryTester tester = new NodeRepositoryTester();
+
+    private final Provisioner provisioner = mock(Provisioner.class);
+    private final NodeRepository nodeRepository = tester.nodeRepository();
+    private final InfrastructureVersions infrastructureVersions = mock(InfrastructureVersions.class);
+    private final InfrastructureProvisioner infrastructureProvisioner = new InfrastructureProvisioner(
+            provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database()));
+
+    @Test
+    public void returns_version_if_usable_nodes_on_old_version() {
+        Version target = Version.fromString("6.123.456");
+        Version oldVersion = Version.fromString("6.122.333");
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.failed, Optional.of(oldVersion));
+        addNode(2, Node.State.dirty, Optional.empty());
+        addNode(3, Node.State.active, Optional.of(oldVersion));
+
+        assertEquals(Optional.of(target), infrastructureProvisioner.getVersionToProvision(NodeType.config));
+    }
+
+    @Test
+    public void returns_version_if_has_usable_nodes_without_version() {
+        Version target = Version.fromString("6.123.456");
+        Version oldVersion = Version.fromString("6.122.333");
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.failed, Optional.of(oldVersion));
+        addNode(2, Node.State.ready, Optional.empty());
+        addNode(3, Node.State.active, Optional.of(target));
+
+        assertEquals(Optional.of(target), infrastructureProvisioner.getVersionToProvision(NodeType.config));
+    }
+
+    @Test
+    public void returns_empty_if_usable_nodes_on_target_version() {
+        Version target = Version.fromString("6.123.456");
+        Version oldVersion = Version.fromString("6.122.333");
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.failed, Optional.of(oldVersion));
+        addNode(2, Node.State.parked, Optional.of(target));
+        addNode(3, Node.State.active, Optional.of(target));
+        addNode(4, Node.State.inactive, Optional.of(target));
+        addNode(5, Node.State.dirty, Optional.empty());
+
+        assertEquals(Optional.empty(), infrastructureProvisioner.getVersionToProvision(NodeType.config));
+    }
+
+    @Test
+    public void returns_empty_if_no_usable_nodes() {
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.of(Version.fromString("6.123.456")));
+
+        // No nodes in node repo
+        assertEquals(Optional.empty(), infrastructureProvisioner.getVersionToProvision(NodeType.config));
+
+        // Add nodes in non-provisionable states
+        addNode(1, Node.State.dirty, Optional.empty());
+        addNode(2, Node.State.failed, Optional.empty());
+        assertEquals(Optional.empty(), infrastructureProvisioner.getVersionToProvision(NodeType.config));
+    }
+
+    @Test
+    public void returns_empty_if_target_version_not_set() {
+        when(infrastructureVersions.getTargetVersionFor(eq(NodeType.config))).thenReturn(Optional.empty());
+        assertEquals(Optional.empty(), infrastructureProvisioner.getVersionToProvision(NodeType.config));
+    }
+
+    private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) {
+        Node node = tester.addNode("id-" + id, "node-" + id, "default", NodeType.config);
+        Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> {
+            ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION;
+            ClusterSpec clusterSpec = ClusterSpec.from(application.getClusterType(), application.getClusterId(), ClusterSpec.Group.from(0), version);
+            ClusterMembership membership = ClusterMembership.from(clusterSpec, 1);
+            Allocation allocation = new Allocation(application.getApplicationId(), membership, new Generation(0, 0), false);
+            return node.with(allocation);
+        });
+        return nodeRepository.database().writeTo(state, nodeWithAllocation.orElse(node), Agent.system, Optional.empty());
+    }
+}