Revert "Revert "Add infrastructure applications to DuperModel""

5 files changed, 224 insertions, 95 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 07e4d586a91..9248ab36f2a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -1,8 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.google.common.collect.ImmutableList;
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Provisioner;
@@ -11,12 +11,8 @@ import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
-import com.yahoo.vespa.service.monitor.application.ConfigServerHostApplication;
-import com.yahoo.vespa.service.monitor.application.ControllerApplication;
-import com.yahoo.vespa.service.monitor.application.ControllerHostApplication;
-import com.yahoo.vespa.service.monitor.application.HostedVespaApplication;
-import com.yahoo.vespa.service.monitor.application.ProxyHostApplication;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.application.InfraApplicationApi;
 
 import java.time.Duration;
 import java.util.List;
@@ -34,33 +30,57 @@ import java.util.stream.Collectors;
 public class InfrastructureProvisioner extends Maintainer {
 
     private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName());
-    private static final List<HostedVespaApplication> HOSTED_VESPA_APPLICATIONS = ImmutableList.of(
-            ConfigServerApplication.CONFIG_SERVER_APPLICATION,
-            ConfigServerHostApplication.CONFIG_SERVER_HOST_APPLICATION,
-            ProxyHostApplication.PROXY_HOST_APPLICATION,
-            ControllerApplication.CONTROLLER_APPLICATION,
-            ControllerHostApplication.CONTROLLER_HOST_APPLICATION);
 
     private final Provisioner provisioner;
     private final InfrastructureVersions infrastructureVersions;
+    private final DuperModelInfraApi duperModel;
 
     public InfrastructureProvisioner(Provisioner provisioner, NodeRepository nodeRepository,
-                                     InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl) {
+                                     InfrastructureVersions infrastructureVersions, Duration interval, JobControl jobControl,
+                                     DuperModelInfraApi duperModel) {
         super(nodeRepository, interval, jobControl);
         this.provisioner = provisioner;
         this.infrastructureVersions = infrastructureVersions;
+        this.duperModel = duperModel;
     }
 
     @Override
     protected void maintain() {
-        for (HostedVespaApplication application: HOSTED_VESPA_APPLICATIONS) {
+        for (InfraApplicationApi application: duperModel.getSupportedInfraApplications()) {
             try (Mutex lock = nodeRepository().lock(application.getApplicationId())) {
-                Optional<Version> version = getTargetVersion(application.getCapacity().type());
-                if (! version.isPresent()) continue;
+                NodeType nodeType = application.getCapacity().type();
+
+                Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType);
+                if (!targetVersion.isPresent()) {
+                    logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set");
+                    duperModel.infraApplicationRemoved(application.getApplicationId());
+                    continue;
+                }
+
+                List<Version> wantedVersions = nodeRepository()
+                        .getNodes(nodeType, Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive)
+                        .stream()
+                        .map(node -> node.allocation()
+                                .map(allocation -> allocation.membership().cluster().vespaVersion())
+                                .orElse(null))
+                        .collect(Collectors.toList());
+                if (wantedVersions.isEmpty()) {
+                    // TODO: Unprovision active nodes from application?
+                    logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision");
+                    duperModel.infraApplicationRemoved(application.getApplicationId());
+                    continue;
+                }
+
+                if (wantedVersions.stream().allMatch(targetVersion.get()::equals) &&
+                        duperModel.infraApplicationIsActive(application.getApplicationId())) {
+                    logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType +
+                            ": Already provisioned to target version " + targetVersion);
+                    continue;
+                }
 
                 List<HostSpec> hostSpecs = provisioner.prepare(
                         application.getApplicationId(),
-                        application.getClusterSpecWithVersion(version.get()),
+                        application.getClusterSpecWithVersion(targetVersion.get()),
                         application.getCapacity(),
                         1, // groups
                         logger::log);
@@ -68,42 +88,14 @@ public class InfrastructureProvisioner extends Maintainer {
                 NestedTransaction nestedTransaction = new NestedTransaction();
                 provisioner.activate(nestedTransaction, application.getApplicationId(), hostSpecs);
                 nestedTransaction.commit();
-            }
-        }
-    }
 
-    /**
-     * Returns the version that the given node type should be provisioned to. This is
-     * the version returned by {@link InfrastructureVersions#getTargetVersionFor} unless a provisioning is:
-     * <ul>
-     *   <li>not possible: no nodes of given type in legal state in node-repo</li>
-     *   <li>redundant: all nodes that can be provisioned already have the right wanted Vespa version</li>
-     * </ul>
-     */
-    Optional<Version> getTargetVersion(NodeType nodeType) {
-        Optional<Version> targetVersion = infrastructureVersions.getTargetVersionFor(nodeType);
-        if (!targetVersion.isPresent()) {
-            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No target version set");
-            return Optional.empty();
-        }
-
-        List<Version> wantedVersions = nodeRepository().getNodes(nodeType,
-                Node.State.ready, Node.State.reserved, Node.State.active, Node.State.inactive).stream()
-                .map(node -> node.allocation()
-                        .map(allocation -> allocation.membership().cluster().vespaVersion())
-                        .orElse(null))
-                .collect(Collectors.toList());
-        if (wantedVersions.isEmpty()) {
-            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType + ": No nodes to provision");
-            return Optional.empty();
-        }
-
-        if (wantedVersions.stream().allMatch(targetVersion.get()::equals)) {
-            logger.log(LogLevel.DEBUG, "Skipping provision of " + nodeType +
-                    ": Already provisioned to target version " + targetVersion);
-            return Optional.empty();
+                duperModel.infraApplicationActivated(
+                        application.getApplicationId(),
+                        hostSpecs.stream().map(HostSpec::hostname).map(HostName::from).collect(Collectors.toList()));
+            } catch (RuntimeException e) {
+                logger.log(LogLevel.INFO, "Failed to activate " + application.getApplicationId(), e);
+                // loop around to activate the next application
+            }
         }
-        return targetVersion;
     }
-
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index ded19a84f0d..677288009da 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareChecker;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorSpareCount;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
 import java.time.Clock;
@@ -57,15 +58,16 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
-                                     ConfigserverConfig configserverConfig) {
+                                     ConfigserverConfig configserverConfig,
+                                     DuperModelInfraApi duperModelInfraApi) {
         this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
-                orchestrator, metric, configserverConfig);
+                orchestrator, metric, configserverConfig, duperModelInfraApi);
     }
 
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
-                                     ConfigserverConfig configserverConfig) {
+                                     ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi) {
         DefaultTimes defaults = new DefaultTimes(zone);
         jobControl = new JobControl(nodeRepository.database());
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
@@ -81,7 +83,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl);
         nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl);
         metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl);
-        infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl);
+        infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl, duperModelInfraApi);
 
 
         RetirementPolicy policy = new RetirementPolicyList(new RetireIPv4OnlyNodes(zone));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index 27215529cf4..035cb8f0aaf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -21,6 +21,7 @@ public class ContainerConfig {
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisioner'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" +
+                       "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
                        "  <component id='com.yahoo.vespa.hosted.provision.lb.LoadBalancerServiceProvider'/>\n" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
new file mode 100644
index 00000000000..b366538949a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.google.inject.Inject;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.application.InfraApplicationApi;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author hakonhall
+ */
+public class MockDuperModel implements DuperModelInfraApi {
+    private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>();
+    private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>();
+
+    @Inject
+    public MockDuperModel() {
+    }
+
+    @Override
+    public List<InfraApplicationApi> getSupportedInfraApplications() {
+        return new ArrayList<>(supportedInfraApps.values());
+    }
+
+    @Override
+    public boolean infraApplicationIsActive(ApplicationId applicationId) {
+        return activeApps.containsKey(applicationId);
+    }
+
+    @Override
+    public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) {
+        activeApps.put(applicationId, hostnames);
+    }
+
+    @Override
+    public void infraApplicationRemoved(ApplicationId applicationId) {
+        activeApps.remove(applicationId);
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
index 642e6adfc75..a4c2cee9404 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisionerTest.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -14,7 +16,8 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.Generation;
 import com.yahoo.vespa.service.monitor.application.ConfigServerApplication;
 import com.yahoo.vespa.service.monitor.application.ControllerApplication;
-import com.yahoo.vespa.service.monitor.application.HostedVespaApplication;
+import com.yahoo.vespa.service.monitor.application.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.application.InfraApplicationApi;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -22,11 +25,16 @@ import org.junit.runners.Parameterized.Parameters;
 
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Optional;
 
-import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
 /**
@@ -38,8 +46,8 @@ public class InfrastructureProvisionerTest {
     @Parameters(name = "application={0}")
     public static Iterable<Object[]> parameters() {
         return Arrays.asList(
-                new HostedVespaApplication[]{ConfigServerApplication.CONFIG_SERVER_APPLICATION},
-                new HostedVespaApplication[]{ControllerApplication.CONTROLLER_APPLICATION}
+                new InfraApplicationApi[]{new ConfigServerApplication()},
+                new InfraApplicationApi[]{new ControllerApplication()}
         );
     }
 
@@ -47,81 +55,161 @@ public class InfrastructureProvisionerTest {
     private final Provisioner provisioner = mock(Provisioner.class);
     private final NodeRepository nodeRepository = tester.nodeRepository();
     private final InfrastructureVersions infrastructureVersions = mock(InfrastructureVersions.class);
+    private final DuperModelInfraApi duperModelInfraApi = mock(DuperModelInfraApi.class);
     private final InfrastructureProvisioner infrastructureProvisioner = new InfrastructureProvisioner(
-            provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database()));
-
-    private final HostedVespaApplication application;
+            provisioner, nodeRepository, infrastructureVersions, Duration.ofDays(99), new JobControl(nodeRepository.database()),
+            duperModelInfraApi);
+    private final HostName node1 = HostName.from("node-1");
+    private final HostName node2 = HostName.from("node-2");
+    private final HostName node3 = HostName.from("node-3");
+    private final Version target = Version.fromString("6.123.456");
+    private final Version oldVersion = Version.fromString("6.122.333");
+
+    private final InfraApplicationApi application;
     private final NodeType nodeType;
 
-    public InfrastructureProvisionerTest(HostedVespaApplication application) {
+    public InfrastructureProvisionerTest(InfraApplicationApi application) {
         this.application = application;
         this.nodeType = application.getCapacity().type();
+        when(duperModelInfraApi.getSupportedInfraApplications()).thenReturn(Collections.singletonList(application));
+    }
+
+    @Test
+    public void remove_application_if_without_target_version() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.empty());
+        addNode(1, Node.State.active, Optional.of(target));
+        infrastructureProvisioner.maintain();
+        verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
+        verifyNoMoreInteractions(provisioner);
+    }
+
+    @Test
+    public void remove_application_if_without_nodes() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+        addNode(1, Node.State.failed, Optional.of(target));
+        addNode(2, Node.State.parked, Optional.empty());
+        infrastructureProvisioner.maintain();
+        verify(duperModelInfraApi).infraApplicationRemoved(application.getApplicationId());
+        verifyNoMoreInteractions(provisioner);
     }
 
     @Test
-    public void returns_version_if_usable_nodes_on_old_version() {
-        Version target = Version.fromString("6.123.456");
-        Version oldVersion = Version.fromString("6.122.333");
+    public void no_op_if_nodes_active_and_on_target_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
         addNode(1, Node.State.failed, Optional.of(oldVersion));
-        addNode(2, Node.State.dirty, Optional.empty());
-        addNode(3, Node.State.active, Optional.of(oldVersion));
+        addNode(2, Node.State.parked, Optional.of(target));
+        addNode(3, Node.State.active, Optional.of(target));
+        addNode(4, Node.State.inactive, Optional.of(target));
+        addNode(5, Node.State.dirty, Optional.empty());
+
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(true);
+
+        infrastructureProvisioner.maintain();
+        verify(duperModelInfraApi, never()).infraApplicationRemoved(any());
+        verify(duperModelInfraApi, never()).infraApplicationActivated(any(), any());
+        verifyNoMoreInteractions(provisioner);
+    }
+
+    @Test
+    public void activates_after_target_has_been_set_the_first_time() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.inactive, Optional.empty());
+        addNode(2, Node.State.parked, Optional.empty());
+        addNode(3, Node.State.active, Optional.empty());
+        addNode(4, Node.State.failed, Optional.empty());
+        addNode(5, Node.State.dirty, Optional.empty());
+
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any())).thenReturn(Arrays.asList(
+                new HostSpec(node1.value(), Collections.emptyList()),
+                new HostSpec(node2.value(), Collections.emptyList()),
+                new HostSpec(node3.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
 
-        assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(nodeType));
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node1, node2, node3));
+    }
+
+
+    @Test
+    public void activates_the_first_time_after_jvm_start() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
+
+        addNode(1, Node.State.active, Optional.of(target));
+
+        when(duperModelInfraApi.infraApplicationIsActive(eq(application.getApplicationId()))).thenReturn(false);
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any())).thenReturn(Collections.singletonList(
+                new HostSpec(node1.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
+
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node1));
     }
 
     @Test
-    public void returns_version_if_has_usable_nodes_without_version() {
-        Version target = Version.fromString("6.123.456");
-        Version oldVersion = Version.fromString("6.122.333");
+    public void provision_usable_nodes_on_old_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
         addNode(1, Node.State.failed, Optional.of(oldVersion));
-        addNode(2, Node.State.ready, Optional.empty());
-        addNode(3, Node.State.active, Optional.of(target));
+        addNode(2, Node.State.inactive, Optional.of(target));
+        addNode(3, Node.State.active, Optional.of(oldVersion));
 
-        assertEquals(Optional.of(target), infrastructureProvisioner.getTargetVersion(nodeType));
+        when(duperModelInfraApi.getSupportedInfraApplications()).thenReturn(Collections.singletonList(application));
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any())).thenReturn(Arrays.asList(
+                new HostSpec(node2.value(), Collections.emptyList()),
+                new HostSpec(node3.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
+
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node2, node3));
     }
 
     @Test
-    public void returns_empty_if_usable_nodes_on_target_version() {
-        Version target = Version.fromString("6.123.456");
-        Version oldVersion = Version.fromString("6.122.333");
+    public void provision_with_usable_node_without_version() {
         when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
         addNode(1, Node.State.failed, Optional.of(oldVersion));
-        addNode(2, Node.State.parked, Optional.of(target));
+        addNode(2, Node.State.ready, Optional.empty());
         addNode(3, Node.State.active, Optional.of(target));
-        addNode(4, Node.State.inactive, Optional.of(target));
-        addNode(5, Node.State.dirty, Optional.empty());
 
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
+        when(provisioner.prepare(any(), any(), any(), anyInt(), any()))
+                .thenReturn(Arrays.asList(
+                        new HostSpec(node2.value(), Collections.emptyList()),
+                        new HostSpec(node3.value(), Collections.emptyList())));
+
+        infrastructureProvisioner.maintain();
+
+        verify(provisioner).prepare(eq(application.getApplicationId()), any(), any(), anyInt(), any());
+        verify(provisioner).activate(any(), eq(application.getApplicationId()), any());
+        verify(duperModelInfraApi).infraApplicationActivated(application.getApplicationId(), Arrays.asList(node2, node3));
     }
 
     @Test
-    public void returns_empty_if_no_usable_nodes() {
-        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(Version.fromString("6.123.456")));
+    public void avoid_provisioning_if_no_usable_nodes() {
+        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.of(target));
 
-        // No nodes in node repo
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
+        infrastructureProvisioner.maintain();
+        verifyNoMoreInteractions(provisioner);
 
         // Add nodes in non-provisionable states
         addNode(1, Node.State.dirty, Optional.empty());
         addNode(2, Node.State.failed, Optional.empty());
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
-    }
 
-    @Test
-    public void returns_empty_if_target_version_not_set() {
-        when(infrastructureVersions.getTargetVersionFor(eq(nodeType))).thenReturn(Optional.empty());
-        assertEquals(Optional.empty(), infrastructureProvisioner.getTargetVersion(nodeType));
+        infrastructureProvisioner.maintain();
+        verifyNoMoreInteractions(provisioner);
     }
 
     private Node addNode(int id, Node.State state, Optional<Version> wantedVespaVersion) {
         Node node = tester.addNode("id-" + id, "node-" + id, "default", nodeType);
         Optional<Node> nodeWithAllocation = wantedVespaVersion.map(version -> {
-            ClusterSpec clusterSpec = ClusterSpec.from(application.getClusterType(), application.getClusterId(), ClusterSpec.Group.from(0), version, false);
+            ClusterSpec clusterSpec = ClusterSpec.from(ClusterSpec.Type.admin, new ClusterSpec.Id("clusterid"), ClusterSpec.Group.from(0), version, false);
             ClusterMembership membership = ClusterMembership.from(clusterSpec, 1);
             Allocation allocation = new Allocation(application.getApplicationId(), membership, new Generation(0, 0), false);
             return node.with(allocation);