Expire inactive load balancers

8 files changed, 220 insertions, 29 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
index effc5b1a41d..4ac3a839ae1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.lb;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Ordering;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer;
 
 import java.util.List;
 import java.util.Objects;
@@ -50,8 +51,8 @@ public class LoadBalancer {
     }
 
     /**
-     * Returns whether this load balancer is inactive. Inactive load balancers cannot be reactivated, and are
-     * eventually deleted
+     * Returns whether this load balancer is inactive. Inactive load balancers cannot be re-activated, and are
+     * eventually removed by {@link LoadBalancerExpirer}.
      */
     public boolean inactive() {
         return inactive;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
index b589e5aed2f..010b983ca12 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java
@@ -17,6 +17,10 @@ public class LoadBalancerServiceMock implements LoadBalancerService {
 
     private final Map<LoadBalancerId, LoadBalancer> loadBalancers = new HashMap<>();
 
+    public Map<LoadBalancerId, LoadBalancer> loadBalancers() {
+        return loadBalancers;
+    }
+
     @Override
     public Protocol protocol() {
         return Protocol.ipv4;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
new file mode 100644
index 00000000000..4b66dff3032
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -0,0 +1,83 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * Periodically remove inactive load balancers permanently.
+ *
+ * When an application is removed, any associated load balancers are only deactivated. This maintainer ensures that
+ * such resources are eventually freed.
+ *
+ * @author mpolden
+ */
+public class LoadBalancerExpirer extends Maintainer {
+
+    private final LoadBalancerService service;
+    private final CuratorDatabaseClient db;
+
+    public LoadBalancerExpirer(NodeRepository nodeRepository, Duration interval, JobControl jobControl,
+                               LoadBalancerService service) {
+        super(nodeRepository, interval, jobControl);
+        this.service = Objects.requireNonNull(service, "service must be non-null");
+        this.db = nodeRepository.database();
+    }
+
+    @Override
+    protected void maintain() {
+        removeInactive();
+    }
+
+    private void removeInactive() {
+        List<LoadBalancerId> failed = new ArrayList<>();
+        Exception lastException = null;
+        try (Lock lock = db.lockLoadBalancers()) {
+            for (LoadBalancerId loadBalancer : inactiveLoadBlancers()) {
+                if (hasNodes(loadBalancer.application())) { // Defer removal if there are still nodes allocated to application
+                    continue;
+                }
+                try {
+                    service.remove(loadBalancer);
+                    db.removeLoadBalancer(loadBalancer);
+                } catch (Exception e) {
+                    failed.add(loadBalancer);
+                    lastException = e;
+                }
+            }
+        }
+        if (!failed.isEmpty()) {
+            log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s",
+                                                    failed.size(),
+                                                    failed.stream()
+                                                          .map(LoadBalancerId::serializedForm)
+                                                          .collect(Collectors.joining(", ")),
+                                                    interval()),
+                    lastException);
+        }
+    }
+
+    private boolean hasNodes(ApplicationId application) {
+        return !nodeRepository().getNodes(application).isEmpty();
+    }
+
+    private List<LoadBalancerId> inactiveLoadBlancers() {
+        return db.readLoadBalancers().entrySet().stream()
+                 .filter(entry -> entry.getValue().inactive())
+                 .map(Map.Entry::getKey)
+                 .collect(Collectors.toList());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index f5576ae00fc..49ede9962eb 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -56,8 +56,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         try {
             if (jobControl.isActive(name()))
                 maintain();
-        }
-        catch (RuntimeException e) {
+        } catch (Throwable e) {
             log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e);
         }
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 946c43ca8fc..2bc60de3c8d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetireIPv4OnlyNodes;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicy;
 import com.yahoo.vespa.hosted.provision.maintenance.retire.RetirementPolicyList;
@@ -50,6 +51,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final NodeRetirer nodeRetirer;
     private final MetricsReporter metricsReporter;
     private final InfrastructureProvisioner infrastructureProvisioner;
+    private final LoadBalancerExpirer loadBalancerExpirer;
 
     private final JobControl jobControl;
     private final InfrastructureVersions infrastructureVersions;
@@ -59,15 +61,17 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
                                      ConfigserverConfig configserverConfig,
-                                     DuperModelInfraApi duperModelInfraApi) {
+                                     DuperModelInfraApi duperModelInfraApi,
+                                     LoadBalancerService loadBalancerService) {
         this(nodeRepository, deployer, provisioner, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
-                orchestrator, metric, configserverConfig, duperModelInfraApi);
+                orchestrator, metric, configserverConfig, duperModelInfraApi, loadBalancerService);
     }
 
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, Provisioner provisioner,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
-                                     ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi) {
+                                     ConfigserverConfig configserverConfig, DuperModelInfraApi duperModelInfraApi,
+                                     LoadBalancerService loadBalancerService) {
         DefaultTimes defaults = new DefaultTimes(zone);
         jobControl = new JobControl(nodeRepository.database());
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
@@ -84,6 +88,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl);
         metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl);
         infrastructureProvisioner = new InfrastructureProvisioner(provisioner, nodeRepository, infrastructureVersions, durationFromEnv("infrastructure_provision_interval").orElse(defaults.infrastructureProvisionInterval), jobControl, duperModelInfraApi);
+        loadBalancerExpirer = new LoadBalancerExpirer(nodeRepository, durationFromEnv("load_balancer_expiry").orElse(defaults.loadBalancerExpiry), jobControl, loadBalancerService);
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
         infrastructureProvisioner.maintain();
@@ -109,6 +114,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         provisionedExpirer.deconstruct();
         metricsReporter.deconstruct();
         infrastructureProvisioner.deconstruct();
+        loadBalancerExpirer.deconstruct();
     }
 
     public JobControl jobControl() { return jobControl; }
@@ -156,6 +162,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration metricsInterval;
         private final Duration retiredInterval;
         private final Duration infrastructureProvisionInterval;
+        private final Duration loadBalancerExpiry;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -171,6 +178,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             metricsInterval = Duration.ofMinutes(1);
             infrastructureProvisionInterval = Duration.ofMinutes(3);
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
+            loadBalancerExpiry = Duration.ofHours(1);
 
             if (zone.environment().equals(Environment.prod) && zone.system() != SystemName.cd) {
                 inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index c99cccd2ab9..ac42bb1a6a5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -456,10 +456,10 @@ public class CuratorDatabaseClient {
         });
     }
 
-    public void removeLoadBalancer(LoadBalancer loadBalancer) {
+    public void removeLoadBalancer(LoadBalancerId loadBalancer) {
         NestedTransaction transaction = new NestedTransaction();
         CuratorTransaction curatorTransaction = newCuratorTransactionIn(transaction);
-        curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer.id()).getAbsolute()));
+        curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer).getAbsolute()));
         transaction.commit();
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
new file mode 100644
index 00000000000..59323bfdeb5
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
@@ -0,0 +1,96 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.component.Vtag;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.hosted.provision.flag.FlagId;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mpolden
+ */
+public class LoadBalancerExpirerTest {
+
+    private ProvisioningTester tester;
+
+    @Before
+    public void before() {
+        tester = new ProvisioningTester(Zone.defaultZone());
+    }
+
+    @Test
+    public void test_maintain() {
+        LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
+                                                              Duration.ofDays(1),
+                                                              new JobControl(tester.nodeRepository().database()),
+                                                              tester.loadBalancerService());
+        tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, true);
+        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+
+        // Deploy two applications with load balancers
+        ClusterSpec.Id cluster = ClusterSpec.Id.from("qrs");
+        ApplicationId app1 = tester.makeApplicationId();
+        ApplicationId app2 = tester.makeApplicationId();
+        LoadBalancerId lb1 = new LoadBalancerId(app1, cluster);
+        LoadBalancerId lb2 = new LoadBalancerId(app2, cluster);
+        deployApplication(app1, cluster);
+        deployApplication(app2, cluster);
+        assertEquals(2, loadBalancers.get().size());
+
+        // Remove one application deactivates load balancers for that application
+        removeApplication(app1);
+        assertTrue(loadBalancers.get().get(lb1).inactive());
+        assertFalse(loadBalancers.get().get(lb2).inactive());
+
+        // Expirer defers removal while nodes are still allocated to application
+        expirer.maintain();
+        assertEquals(2, tester.loadBalancerService().loadBalancers().size());
+
+        // Expirer removes load balancers once nodes are deallocated
+        dirtyNodesOf(app1);
+        expirer.maintain();
+        assertFalse("Inactive load balancer removed", tester.loadBalancerService().loadBalancers().containsKey(lb1));
+
+        // Active load balancer is left alone
+        assertFalse(loadBalancers.get().get(lb2).inactive());
+        assertTrue("Active load balancer is not removed", tester.loadBalancerService().loadBalancers().containsKey(lb2));
+    }
+
+    private void dirtyNodesOf(ApplicationId application) {
+        tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, "unit-test");
+    }
+
+    private void removeApplication(ApplicationId application) {
+        NestedTransaction transaction = new NestedTransaction();
+        tester.provisioner().remove(transaction, application);
+        transaction.commit();
+    }
+
+    private void deployApplication(ApplicationId application, ClusterSpec.Id cluster) {
+        tester.makeReadyNodes(10, "default");
+        List<HostSpec> hosts = tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster,
+                                                                               Vtag.currentVersion, false),
+                                              2, 1,
+                                              "default");
+        tester.activate(application, hosts);
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 5612c8dc665..2c63b9fd62c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -101,7 +101,7 @@ public class ProvisioningTester {
         }
     }
 
-    static FlavorsConfig createConfig() {
+    public static FlavorsConfig createConfig() {
         FlavorConfigBuilder b = new FlavorConfigBuilder();
         b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3);
         b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2);
@@ -170,11 +170,11 @@ public class ProvisioningTester {
         deactivateTransaction.commit();
     }
 
-    Collection<String> toHostNames(Collection<HostSpec> hosts) {
+    public Collection<String> toHostNames(Collection<HostSpec> hosts) {
         return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet());
     }
 
-    Set<String> toHostNames(List<Node> nodes) {
+    public Set<String> toHostNames(List<Node> nodes) {
         return nodes.stream().map(Node::hostname).collect(Collectors.toSet());
     }
 
@@ -182,7 +182,7 @@ public class ProvisioningTester {
      * Asserts that each active node in this application has a restart count equaling the
      * number of matches to the given filters
      */
-    void assertRestartCount(ApplicationId application, HostFilter... filters) {
+    public void assertRestartCount(ApplicationId application, HostFilter... filters) {
         for (Node node : nodeRepository.getNodes(application, Node.State.active)) {
             int expectedRestarts = 0;
             for (HostFilter filter : filters)
@@ -199,7 +199,7 @@ public class ProvisioningTester {
         assertEquals(beforeFailCount + 1, failedNode.status().failCount());
     }
 
-    void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
+    public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
         Set<Integer> indices = new HashSet<>();
         for (HostSpec host : hosts) {
             ClusterSpec nodeCluster = host.membership().get().cluster();
@@ -214,14 +214,14 @@ public class ProvisioningTester {
         assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size());
     }
 
-    HostSpec removeOne(Set<HostSpec> hosts) {
+    public HostSpec removeOne(Set<HostSpec> hosts) {
         Iterator<HostSpec> i = hosts.iterator();
         HostSpec removed = i.next();
         i.remove();
         return removed;
     }
 
-    ApplicationId makeApplicationId() {
+    public ApplicationId makeApplicationId() {
         return ApplicationId.from(
                 TenantName.from(UUID.randomUUID().toString()),
                 ApplicationName.from(UUID.randomUUID().toString()),
@@ -232,15 +232,15 @@ public class ProvisioningTester {
         return makeReadyNodes(n, flavor, NodeType.tenant);
     }
 
-    List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
+    public List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
         return makeReadyNodes(n, flavor, type, 0);
     }
 
-    List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) {
+    public List<Node> makeProvisionedNodes(int count, String flavor, NodeType type, int ipAddressPoolSize) {
         return makeProvisionedNodes(count, flavor, type, ipAddressPoolSize, false);
     }
 
-    List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+    public List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         List<Node> nodes = new ArrayList<>(n);
 
         for (int i = 0; i < n; i++) {
@@ -290,7 +290,7 @@ public class ProvisioningTester {
         return nodes;
     }
 
-    List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) {
+    public List<Node> makeConfigServers(int n, String flavor, Version configServersVersion) {
         List<Node> nodes = new ArrayList<>(n);
         MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver();
 
@@ -322,33 +322,33 @@ public class ProvisioningTester {
         return nodeRepository.getNodes(application.getApplicationId(), Node.State.active);
     }
 
-    List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) {
+    public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize) {
         return makeReadyNodes(n, flavor, type, ipAddressPoolSize, false);
     }
 
-    List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
+    public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int ipAddressPoolSize, boolean dualStack) {
         List<Node> nodes = makeProvisionedNodes(n, flavor, type, ipAddressPoolSize, dualStack);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
 
     /** Creates a set of virtual docker nodes on a single docker host */
-    List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
+    public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
         return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId));
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
+    public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
         return makeReadyVirtualNodes(n, 0, flavor, parentHostId, index -> UUID.randomUUID().toString());
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) {
+    public List<Node> makeReadyVirtualNode(int index, String flavor, String parentHostId) {
         return makeReadyVirtualNodes(1, index, flavor, Optional.of(parentHostId), i -> String.format("node%03d", i));
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId,
+    public List<Node> makeReadyVirtualNodes(int count, int startIndex, String flavor, Optional<String> parentHostId,
                                      Function<Integer, String> nodeNamer) {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = startIndex; i < count + startIndex; i++) {
@@ -362,16 +362,16 @@ public class ProvisioningTester {
         return nodes;
     }
 
-    List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
+    public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
         return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId));
     }
 
     /** Returns the hosts from the input list which are not retired */
-    List<HostSpec> nonRetired(Collection<HostSpec> hosts) {
+    public List<HostSpec> nonRetired(Collection<HostSpec> hosts) {
         return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList());
     }
 
-    void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
+    public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
         long actualNodesWithFlavor = hostSpecs.stream()
                 .map(HostSpec::hostname)
                 .map(this::getNodeFlavor)