Merge pull request #3303 from vespa-engine/smorgrav/fail_on_unresolved_hostname

Fail on unresolved ip adresses

9 files changed, 130 insertions, 67 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 7f595b4a541..6d04685189f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -111,7 +111,10 @@ public class NodeRepository extends AbstractComponent {
 
     /** Returns the Docker image to use for nodes in this */
     public DockerImage dockerImage() { return dockerImage; }
-    
+
+    /** @return The name resolver used to resolve hostname and ip addresses */
+    public NameResolver nameResolver() { return nameResolver; }
+
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
@@ -258,6 +261,7 @@ public class NodeRepository extends AbstractComponent {
         if (ipAddresses.isEmpty()) {
             ipAddresses = nameResolver.getAllByNameOrThrow(hostname);
         }
+
         return Node.create(openStackId, ImmutableSet.copyOf(ipAddresses), additionalIpAddresses, hostname, parentHostname, flavor, type);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java
index 6486343150c..f0165673aa9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java
@@ -2,14 +2,19 @@
 package com.yahoo.vespa.hosted.provision.persistence;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.net.InetAddresses;
+import com.yahoo.log.LogLevel;
 
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
 import javax.naming.directory.Attributes;
 import javax.naming.directory.DirContext;
 import javax.naming.directory.InitialDirContext;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Optional;
 import java.util.Set;
+import java.util.logging.Logger;
 
 /**
  * Implementation of a name resolver that always uses a DNS server to resolve the given name. The intention is to avoid
@@ -19,7 +24,8 @@ import java.util.Set;
  */
 public class DnsNameResolver implements NameResolver {
 
-    /** Resolve IP addresses for given host name */
+    private static Logger logger = Logger.getLogger(DnsNameResolver.class.getName());
+
     @Override
     public Set<String> getAllByNameOrThrow(String hostname) {
         try {
@@ -39,6 +45,18 @@ public class DnsNameResolver implements NameResolver {
         }
     }
 
+    @Override
+    public Optional<String> getHostname(String ipAddress) {
+        try {
+            String hostname = InetAddress.getByName(ipAddress).getHostName();
+            return InetAddresses.isInetAddress(hostname) ? Optional.empty() : Optional.of(hostname);
+        } catch (UnknownHostException e) {
+            // This is not an exceptional state hence the debug level
+            logger.log(LogLevel.DEBUG, "Unable to resolve ipaddress", e);
+        }
+        return Optional.empty();
+    }
+
     private Optional<String> lookupName(String name, Type type) throws NamingException {
         DirContext ctx = new InitialDirContext();
         Attributes attributes = ctx.getAttributes("dns:/" + name, new String[]{type.value});
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java
index b079ebc1100..32451bf4946 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java
@@ -1,15 +1,30 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.persistence;
 
+import java.util.Optional;
 import java.util.Set;
 
 /**
- * Interface for a basic name to IP address resolver.
+ * Interface for a hostname/ip resolver.
  *
  * @author mpolden
  */
 public interface NameResolver {
 
+    /**
+     * Get IP addresses for given host name
+     *
+     * @param hostname The hostname to resolve
+     * @return A set of IPv4 or IPv6 addresses
+     */
     Set<String> getAllByNameOrThrow(String hostname);
 
+    /**
+     * Get hostname from IP address.
+     *
+     * @param ipAddress The IPv4 or IPv6 address for the host
+     * @return Empty if the IP does not resolve or the hostname if it does
+     */
+    Optional<String> getHostname(String ipAddress);
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index c5d60c30356..0bf48b805aa 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -58,7 +58,8 @@ class GroupPreparer {
                                                                   cluster,
                                                                   requestedNodes,
                                                                   nodeRepository.getAvailableFlavors(),
-                                                                  nofSpares);
+                                                                  nofSpares,
+                                                                  nodeRepository.nameResolver());
 
                 prioritizer.addApplicationNodes();
                 prioritizer.addSurplusNodes(surplusActiveNodes);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
index 3df79174b5c..7d5cd85a783 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
@@ -8,9 +8,8 @@ import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.persistence.NameResolver;
 
-import java.net.InetAddress;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -30,7 +29,7 @@ import java.util.stream.Collectors;
  *
  * @author smorgrav
  */
-class NodePrioritizer {
+public class NodePrioritizer {
 
     private final Map<Node, PrioritizableNode> nodes = new HashMap<>();
     private final List<Node> allNodes;
@@ -38,18 +37,18 @@ class NodePrioritizer {
     private final NodeSpec requestedNodes;
     private final ApplicationId appId;
     private final ClusterSpec clusterSpec;
-
+    private final NameResolver nameResolver;
     private final boolean isDocker;
     private final boolean isAllocatingForReplacement;
     private final Set<Node> spareHosts;
     private final Map<Node, ResourceCapacity> headroomHosts;
 
-    NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, NodeFlavors nodeFlavors, int spares) {
+    NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, NodeFlavors nodeFlavors, int spares, NameResolver nameResolver) {
         this.allNodes = Collections.unmodifiableList(allNodes);
         this.requestedNodes = nodeSpec;
         this.clusterSpec = clusterSpec;
         this.appId = appId;
-
+        this.nameResolver = nameResolver;
         this.spareHosts = findSpareHosts(allNodes, spares);
         this.headroomHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors);
 
@@ -73,20 +72,6 @@ class NodePrioritizer {
     }
 
     /**
-     * From ipAddress - get hostname
-     *
-     * @return hostname or null if not able to do the lookup
-     */
-    private static String lookupHostname(String ipAddress) {
-        try {
-            return InetAddress.getByName(ipAddress).getHostName();
-        } catch (UnknownHostException e) {
-            e.printStackTrace();
-        }
-        return null;
-    }
-
-    /**
      * Spare hosts are the two hosts in the system with the most free capacity.
      *
      * We do not count retired or inactive nodes as used capacity (as they could have been
@@ -205,10 +190,10 @@ class NodePrioritizer {
                     Set<String> ipAddresses = DockerHostCapacity.findFreeIps(node, allNodes);
                     if (ipAddresses.isEmpty()) continue;
                     String ipAddress = ipAddresses.stream().findFirst().get();
-                    String hostname = lookupHostname(ipAddress);
-                    if (hostname == null) continue;
+                    Optional<String> hostname = nameResolver.getHostname(ipAddress);
+                    if (!hostname.isPresent()) continue;
                     Node newNode = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress),
-                            Collections.emptySet(), hostname, Optional.of(node.hostname()), getFlavor(requestedNodes), NodeType.tenant);
+                            Collections.emptySet(), hostname.get(), Optional.of(node.hostname()), getFlavor(requestedNodes), NodeType.tenant);
                     PrioritizableNode nodePri = toNodePriority(newNode, false, true);
                     if (!nodePri.violatesSpares || isAllocatingForReplacement) {
                         nodes.put(newNode, nodePri);
@@ -233,7 +218,7 @@ class NodePrioritizer {
     }
 
     /**
-     * Add nodes already provisioned, but not allocatied to any application
+     * Add nodes already provisioned, but not allocated to any application
      */
     void addReadyNodes() {
         allNodes.stream()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java
index b5c10bb7755..d2d2da813d8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java
@@ -10,7 +10,9 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 
 /**
@@ -56,11 +58,25 @@ public class MockNameResolver implements NameResolver {
             return records.get(hostname);
         }
         if (mockAnyLookup) {
-            return Collections.singleton(randomIpAddress());
+            Set<String> ipAddresses = Collections.singleton(randomIpAddress());
+            records.put(hostname, ipAddresses);
+            return ipAddresses;
         }
         throw new RuntimeException(new UnknownHostException("Could not resolve: " + hostname));
     }
 
+    @Override
+    public Optional<String> getHostname(String ipAddress) {
+        for (String host : records.keySet()) {
+            if (records.get(host).contains(ipAddress)) return Optional.of(host);
+        }
+        if (mockAnyLookup) {
+            String hostname = UUID.randomUUID().toString();
+            records.put(hostname, Collections.singleton(ipAddress));
+        }
+        return Optional.empty();
+    }
+
     private static String randomIpAddress() {
         // Generate a random IP in 127/8 (loopback block)
         ThreadLocalRandom random = ThreadLocalRandom.current();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 0885b941401..bcc18cfd876 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -105,6 +105,7 @@ public class MockNodeRepository extends NodeRepository {
         nodes.remove(7);
         nodes = setDirty(nodes);
         setReady(nodes);
+
         fail("host5.yahoo.com", Agent.system, "Failing to unit test");
         setDirty("host55.yahoo.com");
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
index c26865d5690..75d5862f010 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
@@ -32,7 +32,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.is;
@@ -64,7 +63,7 @@ public class DynamicDockerProvisioningTest {
     public void relocate_nodes_from_headroom_hosts() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true));
         enableDynamicAllocation(tester);
-        tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32);
+        tester.makeReadyNodes(4, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
@@ -112,7 +111,7 @@ public class DynamicDockerProvisioningTest {
     public void relocate_nodes_from_spare_hosts() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
         enableDynamicAllocation(tester);
-        tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32);
+        tester.makeReadyNodes(4, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
@@ -158,7 +157,7 @@ public class DynamicDockerProvisioningTest {
     public void new_docker_nodes_are_marked_as_headroom_violations() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true));
         enableDynamicAllocation(tester);
-        tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32);
+        tester.makeReadyNodes(4, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
         Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2");
@@ -214,7 +213,7 @@ public class DynamicDockerProvisioningTest {
     public void only_preferred_container_is_moved_from_hosts_with_headroom_violations() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true));
         enableDynamicAllocation(tester);
-        tester.makeReadyNodes(4, "host", "host-medium", NodeType.host, 32);
+        tester.makeReadyNodes(4, "host-medium", NodeType.host, 32);
         deployZoneApp(tester);
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
         Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2");
@@ -283,7 +282,7 @@ public class DynamicDockerProvisioningTest {
     public void reloacte_failed_nodes() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
         enableDynamicAllocation(tester);
-        tester.makeReadyNodes(5, "host", "host-small", NodeType.host, 32);
+        tester.makeReadyNodes(5, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
@@ -342,7 +341,7 @@ public class DynamicDockerProvisioningTest {
     public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
         enableDynamicAllocation(tester);
-        tester.makeReadyNodes(2, "host", "host-small", NodeType.host, 32);
+        tester.makeReadyNodes(2, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
         List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active);
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
@@ -452,7 +451,7 @@ public class DynamicDockerProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
         enableDynamicAllocation(tester);
 
-        tester.makeProvisionedNodes(3, UUID.randomUUID().toString(), "host-small", NodeType.host, 32);
+        tester.makeProvisionedNodes(3, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
 
         ApplicationId application = tester.makeApplicationId();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 1c82675dbab..d9ce04513a5 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -34,7 +34,6 @@ import java.io.IOException;
 import java.time.temporal.TemporalAmount;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -43,7 +42,6 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -64,6 +62,9 @@ public class ProvisioningTester implements AutoCloseable {
     private final ProvisionLogger provisionLogger;
     private final List<AllocationSnapshot> allocationSnapshots = new ArrayList<>();
 
+    private int nextHost = 0;
+    private int nextIP = 0;
+
     public ProvisioningTester(Zone zone) {
         this(zone, createConfig());
     }
@@ -89,7 +90,7 @@ public class ProvisioningTester implements AutoCloseable {
         }
     }
 
-    public static FlavorsConfig createConfig() {
+    static FlavorsConfig createConfig() {
         FlavorConfigBuilder b = new FlavorConfigBuilder();
         b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3);
         b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2);
@@ -160,11 +161,11 @@ public class ProvisioningTester implements AutoCloseable {
         deactivateTransaction.commit();
     }
 
-    public Set<String> toHostNames(Set<HostSpec> hosts) {
+    Set<String> toHostNames(Set<HostSpec> hosts) {
         return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet());
     }
 
-    public Set<String> toHostNames(List<Node> nodes) {
+    Set<String> toHostNames(List<Node> nodes) {
         return nodes.stream().map(Node::hostname).collect(Collectors.toSet());
     }
 
@@ -172,7 +173,7 @@ public class ProvisioningTester implements AutoCloseable {
      * Asserts that each active node in this application has a restart count equaling the
      * number of matches to the given filters
      */
-    public void assertRestartCount(ApplicationId application, HostFilter... filters) {
+    void assertRestartCount(ApplicationId application, HostFilter... filters) {
         for (Node node : nodeRepository.getNodes(application, Node.State.active)) {
             int expectedRestarts = 0;
             for (HostFilter filter : filters)
@@ -189,7 +190,7 @@ public class ProvisioningTester implements AutoCloseable {
         assertEquals(beforeFailCount + 1, failedNode.status().failCount());
     }
 
-    public void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
+    void assertMembersOf(ClusterSpec requestedCluster, Collection<HostSpec> hosts) {
         Set<Integer> indices = new HashSet<>();
         for (HostSpec host : hosts) {
             ClusterSpec nodeCluster = host.membership().get().cluster();
@@ -204,14 +205,14 @@ public class ProvisioningTester implements AutoCloseable {
         assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size());
     }
 
-    public HostSpec removeOne(Set<HostSpec> hosts) {
+    HostSpec removeOne(Set<HostSpec> hosts) {
         Iterator<HostSpec> i = hosts.iterator();
         HostSpec removed = i.next();
         i.remove();
         return removed;
     }
 
-    public ApplicationId makeApplicationId() {
+    ApplicationId makeApplicationId() {
         return ApplicationId.from(
                 TenantName.from(UUID.randomUUID().toString()),
                 ApplicationName.from(UUID.randomUUID().toString()),
@@ -222,25 +223,48 @@ public class ProvisioningTester implements AutoCloseable {
         return makeReadyNodes(n, flavor, NodeType.tenant);
     }
 
-    public List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
+    List<Node> makeReadyNodes(int n, String flavor, NodeType type) {
         return makeReadyNodes(n, flavor, type, 0);
     }
 
-    public List<Node> makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) {
-        return makeReadyNodes(n, UUID.randomUUID().toString(), flavor, type, additionalIps);
-    }
-
-    public List<Node> makeProvisionedNodes(int n, String prefix, String flavor, NodeType type, int additionalIps) {
+    List<Node> makeProvisionedNodes(int n, String flavor, NodeType type, int additionalIps) {
         List<Node> nodes = new ArrayList<>(n);
+
+
         for (int i = 0; i < n; i++) {
-            Set<String> ips = IntStream.range(additionalIps * i, additionalIps * (i+1))
-                    .mapToObj(j -> String.format("127.0.0.%d", j))
-                    .collect(Collectors.toSet());
-
-            nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(),
-                    prefix + i,
-                    Collections.emptySet(),
-                    ips,
+            nextHost++;
+            nextIP++;
+
+            // One test involves two provision testers - to detect this we check if the
+            // name resolver already contains the next host - if this is the case - bump the indices and move on
+            String testIp = String.format("127.0.0.%d", nextIP);
+            MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver();
+            if (nameResolver.getHostname(testIp).isPresent()) {
+                nextHost += 100;
+                nextIP += 100;
+            }
+
+            String hostname = String.format("host-%d.yahoo.com", nextHost);
+            String ipv4 = String.format("127.0.0.%d", nextIP);
+            String ipv6 = String.format("::%d", nextIP);
+
+            nameResolver.addRecord(hostname, ipv4, ipv6);
+            HashSet<String> hostIps = new HashSet<>();
+            hostIps.add(ipv4);
+            hostIps.add(ipv6);
+
+            Set<String> addips = new HashSet<>();
+            for (int ipSeq = 1; ipSeq < additionalIps; ipSeq++) {
+                nextIP++;
+                String ipv6node = String.format("::%d", nextIP);
+                addips.add(ipv6node);
+                nameResolver.addRecord(String.format("node-%d-of-%s",ipSeq, hostname), ipv6node);
+            }
+
+            nodes.add(nodeRepository.createNode(hostname,
+                    hostname,
+                    hostIps,
+                    addips,
                     Optional.empty(),
                     nodeFlavors.getFlavorOrThrow(flavor),
                     type));
@@ -249,19 +273,19 @@ public class ProvisioningTester implements AutoCloseable {
         return nodes;
     }
 
-    public List<Node> makeReadyNodes(int n, String prefix, String flavor, NodeType type, int additionalIps) {
-        List<Node> nodes = makeProvisionedNodes(n, prefix, flavor, type, additionalIps);
+    List<Node> makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) {
+        List<Node> nodes = makeProvisionedNodes(n, flavor, type, additionalIps);
         nodes = nodeRepository.setDirty(nodes);
         return nodeRepository.setReady(nodes);
     }
 
     /** Creates a set of virtual docker nodes on a single docker host */
-    public List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
+    List<Node> makeReadyDockerNodes(int n, String flavor, String dockerHostId) {
         return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId));
     }
 
     /** Creates a set of virtual nodes on a single parent host */
-    public List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
+    List<Node> makeReadyVirtualNodes(int n, String flavor, Optional<String> parentHostId) {
         List<Node> nodes = new ArrayList<>(n);
         for (int i = 0; i < n; i++) {
             final String hostname = UUID.randomUUID().toString();
@@ -274,16 +298,16 @@ public class ProvisioningTester implements AutoCloseable {
         return nodes;
     }
 
-    public List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
+    List<Node> makeReadyVirtualNodes(int n, String flavor, String parentHostId) {
         return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId));
     }
 
     /** Returns the hosts from the input list which are not retired */
-    public List<HostSpec> nonretired(Collection<HostSpec> hosts) {
+    List<HostSpec> nonretired(Collection<HostSpec> hosts) {
         return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList());
     }
 
-    public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
+    void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
         long actualNodesWithFlavor = hostSpecs.stream()
                 .map(HostSpec::hostname)
                 .map(this::getNodeFlavor)