From 4523019666b2a445394fe7887805af8ce1a603e5 Mon Sep 17 00:00:00 2001 From: toby Date: Wed, 6 Sep 2017 12:52:59 -0700 Subject: Add known records to the mocked name resolver from provision tester --- .../provision/testutils/MockNameResolver.java | 1 + .../DynamicDockerProvisioningTest.java | 15 ++-- .../provisioning/NodePrioritizerTest.java | 22 ------ .../provision/provisioning/ProvisioningTester.java | 84 ++++++++++++++-------- 4 files changed, 62 insertions(+), 60 deletions(-) (limited to 'node-repository/src') diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java index 842fd78ed2c..37b27478dc3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -59,6 +59,7 @@ public class MockNameResolver implements NameResolver { if (mockAnyLookup) { Set ipAddresses = Collections.singleton(randomIpAddress()); records.put(hostname, ipAddresses); + return ipAddresses; } throw new RuntimeException(new UnknownHostException("Could not resolve: " + hostname)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java index 06a0a9b8b45..45f73f2fa43 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.is; @@ -70,7 +69,7 @@ public class DynamicDockerProvisioningTest { public void relocate_nodes_from_headroom_hosts() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); enableDynamicAllocation(tester); - tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32); + tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); @@ -118,7 +117,7 @@ public class DynamicDockerProvisioningTest { public void relocate_nodes_from_spare_hosts() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); enableDynamicAllocation(tester); - tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32); + tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); @@ -164,7 +163,7 @@ public class DynamicDockerProvisioningTest { public void new_docker_nodes_are_marked_as_headroom_violations() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); enableDynamicAllocation(tester); - tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32); + tester.makeReadyNodes(4, "host-small", NodeType.host, 32); deployZoneApp(tester); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2"); @@ -220,7 +219,7 @@ public class DynamicDockerProvisioningTest { public void only_preferred_container_is_moved_from_hosts_with_headroom_violations() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); enableDynamicAllocation(tester); - tester.makeReadyNodes(4, "host", "host-medium", NodeType.host, 32); + tester.makeReadyNodes(4, "host-medium", NodeType.host, 32); deployZoneApp(tester); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2"); @@ -289,7 +288,7 @@ public class DynamicDockerProvisioningTest { public void reloacte_failed_nodes() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); enableDynamicAllocation(tester); - tester.makeReadyNodes(5, "host", "host-small", NodeType.host, 32); + tester.makeReadyNodes(5, "host-small", NodeType.host, 32); deployZoneApp(tester); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); @@ -348,7 +347,7 @@ public class DynamicDockerProvisioningTest { public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); enableDynamicAllocation(tester); - tester.makeReadyNodes(2, "host", "host-small", NodeType.host, 32); + tester.makeReadyNodes(2, "host-small", NodeType.host, 32); deployZoneApp(tester); List dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); @@ -458,7 +457,7 @@ public class DynamicDockerProvisioningTest { ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig()); enableDynamicAllocation(tester); - tester.makeProvisionedNodes(3, UUID.randomUUID().toString(), "host-small", NodeType.host, 32); + tester.makeProvisionedNodes(3, "host-small", NodeType.host, 32); deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java index c7295fd20d0..04c0af5d98a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java @@ -83,26 +83,4 @@ public class NodePrioritizerTest { b.addFlavor("d2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); return b.build(); } - - @Test - public void ignore_hosts_that_is_unresolved_in_dns() { - // Here we are testing the behavoir we need in production - NodePrioritizer.unitTesting = false; - - // Random IP that is not resolving - String hostname = NodePrioritizer.lookupHostname("34.23.12.23"); - Assert.assertNull(hostname); - - // Localhost is always resolving - hostname = NodePrioritizer.lookupHostname("127.0.0.1"); - Assert.assertNotNull(hostname); - - // IPv6 Localhost is always resolving to a hostname - hostname = NodePrioritizer.lookupHostname("::1"); - Assert.assertNotNull(hostname); - - // Unspecified IPv6 should not resolve to a hostname - hostname = NodePrioritizer.lookupHostname("::"); - Assert.assertNull(hostname); - } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 1c82675dbab..d9ce04513a5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.time.temporal.TemporalAmount; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -43,7 +42,6 @@ import java.util.Set; import java.util.UUID; import java.util.logging.Level; import java.util.stream.Collectors; -import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -64,6 +62,9 @@ public class ProvisioningTester implements AutoCloseable { private final ProvisionLogger provisionLogger; private final List allocationSnapshots = new ArrayList<>(); + private int nextHost = 0; + private int nextIP = 0; + public ProvisioningTester(Zone zone) { this(zone, createConfig()); } @@ -89,7 +90,7 @@ public class ProvisioningTester implements AutoCloseable { } } - public static FlavorsConfig createConfig() { + static FlavorsConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); b.addFlavor("small", 1., 2., 50, Flavor.Type.BARE_METAL).cost(2); @@ -160,11 +161,11 @@ public class ProvisioningTester implements AutoCloseable { deactivateTransaction.commit(); } - public Set toHostNames(Set hosts) { + Set toHostNames(Set hosts) { return hosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); } - public Set toHostNames(List nodes) { + Set toHostNames(List nodes) { return nodes.stream().map(Node::hostname).collect(Collectors.toSet()); } @@ -172,7 +173,7 @@ public class ProvisioningTester implements AutoCloseable { * Asserts that each active node in this application has a restart count equaling the * number of matches to the given filters */ - public void assertRestartCount(ApplicationId application, HostFilter... filters) { + void assertRestartCount(ApplicationId application, HostFilter... filters) { for (Node node : nodeRepository.getNodes(application, Node.State.active)) { int expectedRestarts = 0; for (HostFilter filter : filters) @@ -189,7 +190,7 @@ public class ProvisioningTester implements AutoCloseable { assertEquals(beforeFailCount + 1, failedNode.status().failCount()); } - public void assertMembersOf(ClusterSpec requestedCluster, Collection hosts) { + void assertMembersOf(ClusterSpec requestedCluster, Collection hosts) { Set indices = new HashSet<>(); for (HostSpec host : hosts) { ClusterSpec nodeCluster = host.membership().get().cluster(); @@ -204,14 +205,14 @@ public class ProvisioningTester implements AutoCloseable { assertEquals("Indexes in " + requestedCluster + " are disjunct", hosts.size(), indices.size()); } - public HostSpec removeOne(Set hosts) { + HostSpec removeOne(Set hosts) { Iterator i = hosts.iterator(); HostSpec removed = i.next(); i.remove(); return removed; } - public ApplicationId makeApplicationId() { + ApplicationId makeApplicationId() { return ApplicationId.from( TenantName.from(UUID.randomUUID().toString()), ApplicationName.from(UUID.randomUUID().toString()), @@ -222,25 +223,48 @@ public class ProvisioningTester implements AutoCloseable { return makeReadyNodes(n, flavor, NodeType.tenant); } - public List makeReadyNodes(int n, String flavor, NodeType type) { + List makeReadyNodes(int n, String flavor, NodeType type) { return makeReadyNodes(n, flavor, type, 0); } - public List makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) { - return makeReadyNodes(n, UUID.randomUUID().toString(), flavor, type, additionalIps); - } - - public List makeProvisionedNodes(int n, String prefix, String flavor, NodeType type, int additionalIps) { + List makeProvisionedNodes(int n, String flavor, NodeType type, int additionalIps) { List nodes = new ArrayList<>(n); + + for (int i = 0; i < n; i++) { - Set ips = IntStream.range(additionalIps * i, additionalIps * (i+1)) - .mapToObj(j -> String.format("127.0.0.%d", j)) - .collect(Collectors.toSet()); - - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), - prefix + i, - Collections.emptySet(), - ips, + nextHost++; + nextIP++; + + // One test involves two provision testers - to detect this we check if the + // name resolver already contains the next host - if this is the case - bump the indices and move on + String testIp = String.format("127.0.0.%d", nextIP); + MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver(); + if (nameResolver.getHostname(testIp).isPresent()) { + nextHost += 100; + nextIP += 100; + } + + String hostname = String.format("host-%d.yahoo.com", nextHost); + String ipv4 = String.format("127.0.0.%d", nextIP); + String ipv6 = String.format("::%d", nextIP); + + nameResolver.addRecord(hostname, ipv4, ipv6); + HashSet hostIps = new HashSet<>(); + hostIps.add(ipv4); + hostIps.add(ipv6); + + Set addips = new HashSet<>(); + for (int ipSeq = 1; ipSeq < additionalIps; ipSeq++) { + nextIP++; + String ipv6node = String.format("::%d", nextIP); + addips.add(ipv6node); + nameResolver.addRecord(String.format("node-%d-of-%s",ipSeq, hostname), ipv6node); + } + + nodes.add(nodeRepository.createNode(hostname, + hostname, + hostIps, + addips, Optional.empty(), nodeFlavors.getFlavorOrThrow(flavor), type)); @@ -249,19 +273,19 @@ public class ProvisioningTester implements AutoCloseable { return nodes; } - public List makeReadyNodes(int n, String prefix, String flavor, NodeType type, int additionalIps) { - List nodes = makeProvisionedNodes(n, prefix, flavor, type, additionalIps); + List makeReadyNodes(int n, String flavor, NodeType type, int additionalIps) { + List nodes = makeProvisionedNodes(n, flavor, type, additionalIps); nodes = nodeRepository.setDirty(nodes); return nodeRepository.setReady(nodes); } /** Creates a set of virtual docker nodes on a single docker host */ - public List makeReadyDockerNodes(int n, String flavor, String dockerHostId) { + List makeReadyDockerNodes(int n, String flavor, String dockerHostId) { return makeReadyVirtualNodes(n, flavor, Optional.of(dockerHostId)); } /** Creates a set of virtual nodes on a single parent host */ - public List makeReadyVirtualNodes(int n, String flavor, Optional parentHostId) { + List makeReadyVirtualNodes(int n, String flavor, Optional parentHostId) { List nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { final String hostname = UUID.randomUUID().toString(); @@ -274,16 +298,16 @@ public class ProvisioningTester implements AutoCloseable { return nodes; } - public List makeReadyVirtualNodes(int n, String flavor, String parentHostId) { + List makeReadyVirtualNodes(int n, String flavor, String parentHostId) { return makeReadyVirtualNodes(n, flavor, Optional.of(parentHostId)); } /** Returns the hosts from the input list which are not retired */ - public List nonretired(Collection hosts) { + List nonretired(Collection hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); } - public void assertNumberOfNodesWithFlavor(List hostSpecs, String flavor, int expectedCount) { + void assertNumberOfNodesWithFlavor(List hostSpecs, String flavor, int expectedCount) { long actualNodesWithFlavor = hostSpecs.stream() .map(HostSpec::hostname) .map(this::getNodeFlavor) -- cgit v1.2.3