Add infrastructure applications to DuperModel

 DuperModel is (will be) responsible for both active tenant applications
(through SuperModel) and infrastructure applications. This PR is one step
in that direction:
 - All infrastructure applications (config, confighost, controller,
   controllerhost, and proxyhost) are owned and managed by DuperModel.
 - The InfrastructureProvisioner retrieves all possible infra apps from the
   DuperModel (through a reduced API), and "activates" each of them if
   target is set and there are any nodes etc.
 - The InfrastructureProvisioner then notifies the DuperModel which
   apps have been activated, and with which hosts.
 - The DuperModel can then build delegate artificially create ApplicationInfo,
   which gets translated into the application model, and finally the service
   model.
 - The resulting service model has NOT_CHECKED for each hostadmin service
   instance. This is sufficient for goal 1 of this sprint.
 - The config server application currently has health, so that's kept as-is
   for now.
 - Feature flags have been tried and works and allows 1. to disable adding the
   infra apps in the DuperModel, and 2. to enable the infra configserver
   instead of the currently created configserver w/health.

5 files changed, 229 insertions, 93 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 07e4d586a91..d37629241f8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -1,8 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.google.common.collect.ImmutableList;
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Provisioner;
@@ -11,12 +11,9 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
-import com.yahoo.vespa.service.monitor.application.ConfigServerHostApplication;
-import com.yahoo.vespa.service.monitor.application.ControllerApplication;
-import com.yahoo.vespa.service.monitor.application.ControllerHostApplication;
-import com.yahoo.vespa.service.monitor.application.HostedVespaApplication;
-import com.yahoo.vespa.service.monitor.application.ProxyHostApplication;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.application.InfraApplication;
+import com.yahoo.vespa.service.monitor.application.InfraApplicationApi;
 
 import java.time.Duration;
 import java.util.List;
@@ -34,33 +31,56 @@ import java.util.stream.Collectors;
 public class InfrastructureProvisioner extends Maintainer {
 
     private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName());
-    private static final List<HostedVespaApplication> HOSTED_VESPA_APPLICATIONS = ImmutableList.of(
-            ConfigServerApplication.CONFIG_SERVER_APPLICATION,
-            ConfigServerHostApplication.CONFIG_SERVER_HOST_APPLICATION,
-            ProxyHostApplication.PROXY_HOST_APPLICATION,
-            ControllerApplication.CONTROLLER_APPLICATION,
-            ControllerHostApplication.CONTROLLER_HOST_APPLICATION);
 
     private final Provisioner provisioner;
     private final InfrastructureVersions infrastructureVersions;
+    private final DuperModelInfraApi duperModel;
 
     public InfrastructureProvisioner(Provisioner provisioner, NodeRepository nodeRepository,
-                                     InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl) {
+                                     InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl,
+                                     DuperModelInfraApi duperModel) {
         super(nodeRepository, interval, jobControl);
         this.provisioner = provisioner;
         this.infrastructureVersions = infrastructureVersions;
+        this.duperModel = duperModel;
     }
 
     @Override
     protected void maintain() {
-        for (HostedVespaApplication application: HOSTED_VESPA_APPLICATIONS) {
+        for (InfraApplicationApi application: duperModel.getSupportedInfraApplications()) {
             try (Mutex lock = nodeRepository().lock(application.getApplicationId())) {
-                Optional<Version> version = getTargetVersion(application.getCapacity().type());
-                if (! version.isPresent()) continue;
+                NodeType nodeType = application.getCapacity().type();
+
+                Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType);
+                if (!targetVersion.isPresent()) {
+                    logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set");
+                    duperModel.infraApplicationRemoved(application.getApplicationId());
+                    continue;
+                }
+
+                List<Version> wantedVersions = nodeRepository()
+                        .getNodes(nodeType, Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive)
+                        .stream()
+                        .map(node -> node.allocation()
+                                .map(allocation -> allocation.membership().cluster().vespaVersion())
+                                .orElse(null))
+                        .collect(Collectors.toList());
+                if (wantedVersions.isEmpty()) {
+                    logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision");
+                    duperModel.infraApplicationRemoved(application.getApplicationId());
+                    continue;
+                }
+
+                if (wantedVersions.stream().allMatch(targetVersion.get()::equals) &&
+                        duperModel.infraApplicationIsActive(application.getApplicationId())) {
+                    logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType +
+                            ": Already provisioned to target version " + targetVersion);
+                    continue;
+                }
 
                 List<HostSpec> hostSpecs = provisioner.prepare(
                         application.getApplicationId(),
-                        application.getClusterSpecWithVersion(version.get()),
+                        application.getClusterSpecWithVersion(targetVersion.get()),
                         application.getCapacity(),
                         1, // groups
                         logger::log);
@@ -68,42 +88,14 @@ public class InfrastructureProvisioner extends Maintainer {
                 NestedTransaction nestedTransaction = new NestedTransaction();
                 provisioner.activate(nestedTransaction, application.getApplicationId(), hostSpecs);
                 nestedTransaction.commit();
-            }
-        }
-    }
 
-    /**
-     * Returns the version that the given node type should be provisioned to. This is
-     * the version returned by {@link InfrastructureVersions#getTargetVersionFor} unless a provisioning is:
-     * <ul>
-     *   <li>not possible: no nodes of given type in legal state in node-repo</li>
-     *   <li>redundant: all nodes that can be provisioned already have the right wanted Vespa version</li>
-     * </ul>
-     */
-    Optional<Version> getTargetVersion(NodeType nodeType) {
-        Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType);
-        if (!targetVersion.isPresent()) {
-            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set");
-            return Optional.empty();
-        }
-
-        List<Version> wantedVersions = nodeRepository().getNodes(nodeType,
-                Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive).stream()
-                .map(node -> node.allocation()
-                        .map(allocation -> allocation.membership().cluster().vespaVersion())
-                        .orElse(null))
-                .collect(Collectors.toList());
-        if (wantedVersions.isEmpty()) {
-            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision");
-            return Optional.empty();
-        }
-
-        if (wantedVersions.stream().allMatch(targetVersion.get()::equals)) {
-            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType +
-                    ": Already provisioned to target version " + targetVersion);
-            return Optional.empty();
+                duperModel.infraApplicationActivated(
+                        application.getApplicationId(),
+                        hostSpecs.stream().map(HostSpec::hostname).map(HostName::from).collect(Collectors.toList()));
+            } catch (RuntimeException e) {
+                logger.log(LogLevel.INFO, "Failed to activate " + application.getApplicationId(), e);
+                // loop around to activate the next application
+            }
         }
-        return targetVersion;
     }
-
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index ded19a84f0d..677288009da 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareChecker;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareCount;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
 import java.time.Clock;
@@ -57,15 +58,16 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
-                                     ConfigserverConfig configserverConfig) {
+                                     ConfigserverConfig configserverConfig,
+                                     DuperModelInfraApi duperModelInfraApi) {
         this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
-                orchestrator, metric, configserverConfig);
+                orchestrator, metric, configserverConfig, duperModelInfraApi);
     }
 
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
-                                     ConfigserverConfig configserverConfig) {
+                                     ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi) {
         DefaultTimes defaults = new DefaultTimes(zone);
         jobControl = new JobControl(nodeRepository.database());
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
@@ -81,7 +83,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl);
         nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl);
         metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl);
-        infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl);
+        infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl, duperModelInfraApi);
 
 
         RetirementPolicy policy = new RetirementPolicyList(new RetireIPv4OnlyNodes(zone));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index 27215529cf4..035cb8f0aaf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -21,6 +21,7 @@ public class ContainerConfig {
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" +
+                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceProvider'/>\n" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
new file mode 100644
index 00000000000..b366538949a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.google.inject.Inject;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.application.InfraApplicationApi;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author hakonhall
+ */
+public class MockDuperModel implements DuperModelInfraApi {
+    private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>();
+    private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>();
+
+    @Inject
+    public MockDuperModel() {
+    }
+
+    @Override
+    public List<InfraApplicationApi> getSupportedInfraApplications() {
+        return new ArrayList<>(supportedInfraApps.values());
+    }
+
+    @Override
+    public boolean infraApplicationIsActive(ApplicationId applicationId) {
+        return activeApps.containsKey(applicationId);
+    }
+
+    @Override
+    public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) {
+        activeApps.put(applicationId, hostnames);
+    }
+
+    @Override
+    public void infraApplicationRemoved(ApplicationId applicationId) {
+        activeApps.remove(applicationId);
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
index 642e6adfc75..4926d3f5013 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
@@ -4,8 +4,11 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Provisioner;
+import com.yahoo.log.event.Collection;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
@@ -14,7 +17,10 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
 import com.yahoo.vespa.service.monitor.application.ControllerApplication;
-import com.yahoo.vespa.service.monitor.application.HostedVespaApplication;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.application.InfraApplication;
+import com.yahoo.vespa.service.monitor.application.InfraApplicationApi;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -22,11 +28,20 @@ import org.junit.runners.Parameterized.Parameters;
 
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyList;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
 /**
@@ -38,8 +53,8 @@ public class InfrastructureProvisionerTest {
     @Parameters(name = "application={0}")
     public static Iterable<Object[]> parameters() {
         return Arrays.asList(
-                new HostedVespaApplication[]{ConfigServerApplication.CONFIG_SERVER_APPLICATION},
-                new HostedVespaApplication[]{ControllerApplication.CONTROLLER_APPLICATION}
+                new InfraApplicationApi[]{new ConfigServerApplication()},
+                new InfraApplicationApi[]{new ControllerApplication()}
         );
     }
 
@@ -47,81 +62,161 @@ public class InfrastructureProvisionerTest {
     private final Provisioner provisioner = mock(Provisioner.class);
     private final NodeRepository nodeRepository = tester.nodeRepository();
     private final InfrastructureVersions infrastructureVersions = mock(InfrastructureVersions.class);
+    private final DuperModelInfraApi duperModelInfraApi = mock(DuperModelInfraApi.class);
     private final InfrastructureProvisioner infrastructureProvisioner = new InfrastructureProvisioner(
-            provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database()));
-
-    private final HostedVespaApplication application;
+            provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database()),
+            duperModelInfraApi);
+    private final HostName node1 = HostName.from("node-1");
+    private final HostName node2 = HostName.from("node-2");
+    private final HostName node3 = HostName.from("node-3");
+    private final Version target = Version.fromString("6.123.456");
+    private final Version oldVersion = Version.fromString("6.122.333");
+
+    private final InfraApplicationApi application;
     private final NodeType nodeType;
 
-    public InfrastructureProvisionerTest(HostedVespaApplication application) {
+    public InfrastructureProvisionerTest(InfraApplicationApi application) {
         this.application = application;
         this.nodeType = application.getCapacity().type();
+        when(duperModelInfraApi.getSupportedInfraApplications()).thenReturn(Collections.singletonList(application));
+    }
+
+    @Test
+    public void remove_application_if_without_target_version() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.empty());
+        addNode(1, Node.State.active, Optional.of(target));
+        infrastructureProvisioner.maintain();
+        verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
+        verifyNoMoreInteractions(provisioner);
+    }
+
+    @Test
+    public void remove_application_if_without_nodes() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+        addNode(1, Node.State.failed, Optional.of(target));
+        addNode(2, Node.State.parked, Optional.empty());
+        infrastructureProvisioner.maintain();
+        verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
+        verifyNoMoreInteractions(provisioner);
     }
 
     @Test
-    public void returns_version_if_usable_nodes_on_old_version() {
-        Version target = Version.fromString("6.123.456");
-        Version oldVersion = Version.fromString("6.122.333");
+    public void no_op_if_nodes_active_and_on_target_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
         addNode(1, Node.State.failed, Optional.of(oldVersion));
-        addNode(2, Node.State.dirty, Optional.empty());
-        addNode(3, Node.State.active, Optional.of(oldVersion));
+        addNode(2, Node.State.parked, Optional.of(target));
+        addNode(3, Node.State.active, Optional.of(target));
+        addNode(4, Node.State.inactive, Optional.of(target));
+        addNode(5, Node.State.dirty, Optional.empty());
+
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true);
+
+        infrastructureProvisioner.maintain();
+        verify(duperModelInfraApi, never()).infraApplicationRemoved(any());
+        verify(duperModelInfraApi, never()).infraApplicationActivated(any(), any());
+        verifyNoMoreInteractions(provisioner);
+    }
+
+    @Test
+    public void activates_after_target_has_been_set_the_first_time() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.inactive, Optional.empty());
+        addNode(2, Node.State.parked, Optional.empty());
+        addNode(3, Node.State.active, Optional.empty());
+        addNode(4, Node.State.failed, Optional.empty());
+        addNode(5, Node.State.dirty, Optional.empty());
+
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any())).thenReturn(Arrays.asList(
+                new HostSpec(node1.value(), Collections.emptyList()),
+                new HostSpec(node2.value(), Collections.emptyList()),
+                new HostSpec(node3.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
 
-        assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(nodeType));
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node1, node2, node3));
+    }
+
+
+    @Test
+    public void activates_the_first_time_after_jvm_start() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.active, Optional.of(target));
+
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(false);
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any())).thenReturn(Collections.singletonList(
+                new HostSpec(node1.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
+
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node1));
     }
 
     @Test
-    public void returns_version_if_has_usable_nodes_without_version() {
-        Version target = Version.fromString("6.123.456");
-        Version oldVersion = Version.fromString("6.122.333");
+    public void provision_usable_nodes_on_old_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
         addNode(1, Node.State.failed, Optional.of(oldVersion));
-        addNode(2, Node.State.ready, Optional.empty());
-        addNode(3, Node.State.active, Optional.of(target));
+        addNode(2, Node.State.inactive, Optional.of(target));
+        addNode(3, Node.State.active, Optional.of(oldVersion));
 
-        assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(nodeType));
+        when(duperModelInfraApi.getSupportedInfraApplications()).thenReturn(Collections.singletonList(application));
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any())).thenReturn(Arrays.asList(
+                new HostSpec(node2.value(), Collections.emptyList()),
+                new HostSpec(node3.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
+
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node2, node3));
     }
 
     @Test
-    public void returns_empty_if_usable_nodes_on_target_version() {
-        Version target = Version.fromString("6.123.456");
-        Version oldVersion = Version.fromString("6.122.333");
+    public void provision_with_usable_node_without_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
         addNode(1, Node.State.failed, Optional.of(oldVersion));
-        addNode(2, Node.State.parked, Optional.of(target));
+        addNode(2, Node.State.ready, Optional.empty());
         addNode(3, Node.State.active, Optional.of(target));
-        addNode(4, Node.State.inactive, Optional.of(target));
-        addNode(5, Node.State.dirty, Optional.empty());
 
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any()))
+                .thenReturn(Arrays.asList(
+                        new HostSpec(node2.value(), Collections.emptyList()),
+                        new HostSpec(node3.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
+
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node2, node3));
     }
 
     @Test
     public void returns_empty_if_no_usable_nodes() {
-        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(Version.fromString("6.123.456")));
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
-        // No nodes in node repo
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
+        infrastructureProvisioner.maintain();
+        verifyNoMoreInteractions(provisioner);
 
         // Add nodes in non-provisionable states
         addNode(1, Node.State.dirty, Optional.empty());
         addNode(2, Node.State.failed, Optional.empty());
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
-    }
 
-    @Test
-    public void returns_empty_if_target_version_not_set() {
-        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.empty());
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
+        infrastructureProvisioner.maintain();
+        verifyNoMoreInteractions(provisioner);
     }
 
     private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) {
         Node node = tester.addNode("id-" + id, "node-" + id, "default", nodeType);
         Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> {
-            ClusterSpec clusterSpec = ClusterSpec.from(application.getClusterType(), application.getClusterId(), ClusterSpec.Group.from(0), version, false);
+            ClusterSpec clusterSpec = ClusterSpec.from(ClusterSpec.Type.admin, new ClusterSpec.Id("clusterid"), ClusterSpec.Group.from(0), version, false);
             ClusterMembership membership = ClusterMembership.from(clusterSpec, 1);
             Allocation allocation = new Allocation(application.getApplicationId(), membership, new Generation(0, 0), false);
             return node.with(allocation);