diff options
| author | Martin Polden <mpolden@mpolden.no> | 2020-10-22 13:05:09 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-10-22 13:05:09 +0200 |
| commit | 001b0d91959381cff71ac1dcf0fd2baff7d023a9 (patch) | |
| tree | d4392dd4e337ef53c864eebb6537eb07e2834749 /node-repository/src | |
| parent | 039efbbf13562a8593d45d23a0684965345ab9d8 (diff) | |
| parent | 39591a2bc7fc65ecfca542408dceed009ea761dd (diff) | |
Merge pull request #15004 from vespa-engine/mpolden/wait-for-dns
Wait for DNS to resolve for dynamically provisioned hosts
Diffstat (limited to 'node-repository/src')
20 files changed, 319 insertions, 558 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 21f28f8385a..e288f08a681 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -89,7 +89,7 @@ public final class Node implements Nodelike { requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); if (parentHostname.isPresent()) { - if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); + if (!ipConfig.pool().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 0136c9bcd3b..fd7dbc9716b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -49,7 +49,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.LinkedHashSet; @@ -381,9 +380,9 @@ public class NodeRepository extends AbstractComponent { if (children) { return candidates.childrenOf(node).asList().stream() .map(childNode -> getNodeAcl(childNode, candidates)) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); + .collect(Collectors.toUnmodifiableList()); } - return Collections.singletonList(getNodeAcl(node, candidates)); + return List.of(getNodeAcl(node, candidates)); } // ----------------- Node lifecycle ----------------------------------------------------------- @@ -392,7 +391,7 @@ public class NodeRepository extends AbstractComponent { public Node createNode(String openStackId, String hostname, IP.Config ipConfig, Optional<String> parentHostname, Flavor flavor, Optional<TenantName> reservedTo, NodeType type) { if (ipConfig.primary().isEmpty()) // TODO: Remove this. Only test code hits this path - ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname)); + ipConfig = ipConfig.with(nameResolver.resolveAll(hostname)); return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type, Optional.empty()); } @@ -461,6 +460,8 @@ public class NodeRepository extends AbstractComponent { .map(node -> { if (node.state() != State.provisioned && node.state() != State.dirty) illegal("Can not set " + node + " ready. It is not provisioned or dirty."); + if (node.type() == NodeType.host && node.ipConfig().pool().isEmpty()) + illegal("Can not set host " + node + " ready. Its IP address pool is empty."); return node.withWantToRetire(false, false, Agent.system, clock.instant()); }) .collect(Collectors.toList()); @@ -474,7 +475,7 @@ public class NodeRepository extends AbstractComponent { new NoSuchNodeException("Could not move " + hostname + " to ready: Node not found")); if (nodeToReady.state() == State.ready) return nodeToReady; - return setReady(Collections.singletonList(nodeToReady), agent, reason).get(0); + return setReady(List.of(nodeToReady), agent, reason).get(0); } /** Reserve nodes. This method does <b>not</b> lock the node repository */ @@ -658,7 +659,7 @@ public class NodeRepository extends AbstractComponent { if ( ! failureReasons.isEmpty()) illegal(node + " cannot be readied because it has hard failures: " + failureReasons); - return setReady(Collections.singletonList(node), agent, reason).get(0); + return setReady(List.of(node), agent, reason).get(0); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 8efe6606c76..0f70542f141 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator; @@ -79,6 +80,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { List<Node> updatedNodes = hostProvisioner.provision(host, children); + verifyDns(updatedNodes); nodeRepository().write(updatedNodes, lock); } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Failed to provision " + host.hostname() + " with " + children.size() + " children: " + @@ -168,6 +170,15 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toList()); } + /** Verify DNS configuration of given nodes */ + private void verifyDns(List<Node> nodes) { + for (var node : nodes) { + for (var ipAddress : node.ipConfig().primary()) { + IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver()); + } + } + } + /** Returns hosts that are considered available, i.e. not parked or flagged for deprovisioning */ private static List<Node> availableHostsOf(NodeList nodes) { return nodes.hosts() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index f64f27b1219..c76239527bd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.config.provision.Flavor; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -44,7 +43,7 @@ public class NodeRebooter extends NodeRepositoryMaintainer { protected boolean maintain() { // Reboot candidates: Nodes in long-term states, where we know we can safely orchestrate a reboot List<Node> nodesToReboot = nodeRepository().getNodes(Node.State.active, Node.State.ready).stream() - .filter(node -> node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) + .filter(node -> node.type().isHost()) .filter(this::shouldReboot) .collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 321301a46a4..6b2e1da0432 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; +import com.yahoo.vespa.hosted.provision.persistence.NameResolver.RecordType; import java.net.InetAddress; import java.util.Collections; @@ -257,6 +258,10 @@ public class IP { return addresses.asSet(); } + public boolean isEmpty() { + return asSet().isEmpty(); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -335,8 +340,8 @@ public class IP { * @return An allocation containing 1 IPv6 address and 1 IPv4 address (if hostname is dual-stack) */ private static Allocation ofIpv6(String ipAddress, NameResolver resolver) { - String hostname6 = resolver.getHostname(ipAddress).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipAddress)); - List<String> ipv4Addresses = resolver.getAllByNameOrThrow(hostname6).stream() + String hostname6 = resolver.resolveHostname(ipAddress).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipAddress)); + List<String> ipv4Addresses = resolver.resolveAll(hostname6).stream() .filter(IP::isV4) .collect(Collectors.toList()); if (ipv4Addresses.size() > 1) { @@ -344,7 +349,7 @@ public class IP { } Optional<String> ipv4Address = ipv4Addresses.stream().findFirst(); ipv4Address.ifPresent(addr -> { - String hostname4 = resolver.getHostname(addr).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + addr)); + String hostname4 = resolver.resolveHostname(addr).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + addr)); if (!hostname6.equals(hostname4)) { throw new IllegalArgumentException(String.format("Hostnames resolved from each IP address do not " + "point to the same hostname [%s -> %s, %s -> %s]", @@ -362,8 +367,8 @@ public class IP { * @return An allocation containing 1 IPv4 address. */ private static Allocation ofIpv4(String ipAddress, NameResolver resolver) { - String hostname4 = resolver.getHostname(ipAddress).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipAddress)); - List<String> addresses = resolver.getAllByNameOrThrow(hostname4).stream() + String hostname4 = resolver.resolveHostname(ipAddress).orElseThrow(() -> new IllegalArgumentException("Could not resolve IP address: " + ipAddress)); + List<String> addresses = resolver.resolveAll(hostname4).stream() .filter(IP::isV4) .collect(Collectors.toList()); if (addresses.size() != 1) { @@ -403,7 +408,7 @@ public class IP { } - /** Validate IP address*/ + /** Parse given IP address string */ public static InetAddress parse(String ipAddress) { try { return InetAddresses.forString(ipAddress); @@ -412,6 +417,23 @@ public class IP { } } + /** Verify DNS configuration of given hostname and IP address */ + public static void verifyDns(String hostname, String ipAddress, NameResolver resolver) { + RecordType recordType = isV6(ipAddress) ? RecordType.AAAA : RecordType.A; + Set<String> addresses = resolver.resolve(hostname, recordType); + if (!addresses.equals(Set.of(ipAddress))) + throw new IllegalArgumentException("Expected " + hostname + " to resolve to " + ipAddress + + ", but got " + addresses); + + Optional<String> reverseHostname = resolver.resolveHostname(ipAddress); + if (reverseHostname.isEmpty()) + throw new IllegalArgumentException(ipAddress + " did not resolve to a hostname"); + + if (!reverseHostname.get().equals(hostname)) + throw new IllegalArgumentException(ipAddress + " resolved to " + reverseHostname.get() + + ", which does not match expected hostname " + hostname); + } + /** Convert IP address to string. This uses :: for zero compression in IPv6 addresses. */ public static String asString(InetAddress inetAddress) { return InetAddresses.toAddrString(inetAddress); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java index 6d2c074ab36..1367e4d0884 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java @@ -1,9 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; -import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; -import java.util.logging.Level; import javax.naming.NamingException; import javax.naming.directory.Attribute; @@ -12,9 +10,12 @@ import javax.naming.directory.DirContext; import javax.naming.directory.InitialDirContext; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.logging.Logger; /** * Implementation of a name resolver that always uses a DNS server to resolve the given name. The intention is to avoid @@ -24,60 +25,46 @@ import java.util.logging.Logger; */ public class DnsNameResolver implements NameResolver { - private static final Logger logger = Logger.getLogger(DnsNameResolver.class.getName()); + @Override + public Set<String> resolveAll(String name) { + return resolve(name, RecordType.A, RecordType.AAAA); + } @Override - public Set<String> getAllByNameOrThrow(String hostname) { - try { - Optional<String> cname = lookupName(hostname, Type.CNAME); - if (cname.isPresent()) { - hostname = cname.get(); + public Set<String> resolve(String name, RecordType first, RecordType... rest) { + Set<String> results = new HashSet<>(); + for (var type : EnumSet.of(first, rest)) { + try { + results.addAll(lookupName(name, type)); + } catch (Exception e) { + throw new RuntimeException(e); } - Optional<String> inet4Address = lookupName(hostname, Type.A); - Optional<String> inet6Address = lookupName(hostname, Type.AAAA); - - ImmutableSet.Builder<String> ipAddresses = ImmutableSet.builder(); - inet4Address.ifPresent(ipAddresses::add); - inet6Address.ifPresent(ipAddresses::add); - return ipAddresses.build(); - } catch (Exception e) { - throw new RuntimeException(e); } + return Collections.unmodifiableSet(results); } @Override - public Optional<String> getHostname(String ipAddress) { + public Optional<String> resolveHostname(String ipAddress) { try { + // TODO(mpolden): Use lookupName instead. IP address must be translated to its reverse + // notation first. String hostname = InetAddress.getByName(ipAddress).getHostName(); return InetAddresses.isInetAddress(hostname) ? Optional.empty() : Optional.of(hostname); - } catch (UnknownHostException e) { - // This is not an exceptional state hence the debug level - logger.log(Level.FINE, "Unable to resolve ipaddress", e); + } catch (UnknownHostException ignored) { } return Optional.empty(); } - private Optional<String> lookupName(String name, Type type) throws NamingException { + private Set<String> lookupName(String name, RecordType type) throws NamingException { DirContext ctx = new InitialDirContext(); - Attributes attributes = ctx.getAttributes("dns:/" + name, new String[]{type.value}); - Optional<Attribute> attribute = Optional.ofNullable(attributes.get(type.value)); - if (attribute.isPresent()) { - return Optional.ofNullable(attribute.get().get()).map(Object::toString); - } - return Optional.empty(); - } - - private enum Type { - - A("A"), - AAAA("AAAA"), - CNAME("CNAME"); - - private final String value; - - Type(String value) { - this.value = value; + Attributes attributes = ctx.getAttributes("dns:/" + name, new String[]{type.value()}); + Attribute attribute = attributes.get(type.value()); + if (attribute == null) { + return Set.of(); } + Set<String> results = new HashSet<>(); + attribute.getAll().asIterator().forEachRemaining(value -> results.add(Objects.toString(value))); + return results; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java index 32451bf4946..31e313b6706 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NameResolver.java @@ -5,26 +5,37 @@ import java.util.Optional; import java.util.Set; /** - * Interface for a hostname/ip resolver. + * Interface for a basic DNS name resolver. * * @author mpolden */ public interface NameResolver { - /** - * Get IP addresses for given host name - * - * @param hostname The hostname to resolve - * @return A set of IPv4 or IPv6 addresses - */ - Set<String> getAllByNameOrThrow(String hostname); - - /** - * Get hostname from IP address. - * - * @param ipAddress The IPv4 or IPv6 address for the host - * @return Empty if the IP does not resolve or the hostname if it does - */ - Optional<String> getHostname(String ipAddress); + /** Resolve {@link RecordType#A} and {@link RecordType#AAAA} records for given name */ + Set<String> resolveAll(String name); + + /** Resolve given record type(s) for given name */ + Set<String> resolve(String name, RecordType first, RecordType... rest); + + /** Resolve hostname from given IP address, if any exist */ + Optional<String> resolveHostname(String ipAddress); + + /** DNS record types */ + enum RecordType { + + A("A"), + AAAA("AAAA"); + + private final String value; + + public String value() { + return value; + } + + RecordType(String value) { + this.value = value; + } + + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java index 9b524d08160..1b76d02dfd1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNameResolver.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import java.net.UnknownHostException; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -34,8 +33,8 @@ public class MockNameResolver implements NameResolver { public MockNameResolver addRecord(String hostname, String... ipAddress) { Objects.requireNonNull(hostname, "hostname must be non-null"); Arrays.stream(ipAddress).forEach(ip -> Objects.requireNonNull(ip, "ipAddress must be non-null")); - records.putIfAbsent(hostname, new HashSet<>()); - records.get(hostname).addAll(Arrays.asList(ipAddress)); + records.computeIfAbsent(hostname, (k) -> new HashSet<>()) + .addAll(Arrays.asList(ipAddress)); return this; } @@ -51,7 +50,7 @@ public class MockNameResolver implements NameResolver { /** * When true, only records added with {@link MockNameResolver#addReverseRecord(String, String)} are considered by - * {@link MockNameResolver#getHostname(String)}. Otherwise the latter returns the IP address by reversing the lookup + * {@link MockNameResolver#resolveHostname(String)}. Otherwise the latter returns the IP address by reversing the lookup * implicitly. */ public MockNameResolver explicitReverseRecords() { @@ -60,20 +59,25 @@ public class MockNameResolver implements NameResolver { } @Override - public Set<String> getAllByNameOrThrow(String hostname) { - if (records.containsKey(hostname)) { - return records.get(hostname); + public Set<String> resolveAll(String name) { + if (records.containsKey(name)) { + return records.get(name); } if (mockAnyLookup) { - Set<String> ipAddresses = Collections.singleton(randomIpAddress()); - records.put(hostname, ipAddresses); + Set<String> ipAddresses = Set.of(randomIpAddress()); + records.put(name, ipAddresses); return ipAddresses; } - throw new RuntimeException(new UnknownHostException("Could not resolve: " + hostname)); + throw new RuntimeException(new UnknownHostException("Could not resolve: " + name)); } @Override - public Optional<String> getHostname(String ipAddress) { + public Set<String> resolve(String name, RecordType first, RecordType... rest) { + return resolveAll(name); + } + + @Override + public Optional<String> resolveHostname(String ipAddress) { if (!explicitReverseRecords) { return records.entrySet().stream() .filter(kv -> kv.getValue().contains(ipAddress)) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 5c80cd760e4..b817c4b58b5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -43,6 +43,7 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -200,6 +201,24 @@ public class DynamicProvisioningMaintainerTest { tester.assertNodesUnchanged(); } + @Test + public void defer_writing_ip_addresses_until_dns_resolves() { + var tester = new DynamicProvisioningTester().addInitialNodes(); + tester.hostProvisioner.with(Behaviour.failDnsUpdate); + + Supplier<List<Node>> provisioning = () -> tester.nodeRepository.getNodes(NodeType.host, Node.State.provisioned); + assertEquals(2, provisioning.get().size()); + tester.maintainer.maintain(); + + assertTrue("No IP addresses written as DNS updates are failing", + provisioning.get().stream().allMatch(host -> host.ipConfig().pool().isEmpty())); + + tester.hostProvisioner.without(Behaviour.failDnsUpdate); + tester.maintainer.maintain(); + assertTrue("IP addresses written as DNS updates are succeeding", + provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().isEmpty())); + } + private static class DynamicProvisioningTester { private static final ApplicationId tenantApp = ApplicationId.from("mytenant", "myapp", "default"); @@ -222,7 +241,7 @@ public class DynamicProvisioningMaintainerTest { } public DynamicProvisioningTester(Cloud cloud) { - MockNameResolver nameResolver = new MockNameResolver().mockAnyLookup(); + MockNameResolver nameResolver = new MockNameResolver(); this.hostProvisioner = new MockHostProvisioner(flavors, nameResolver); this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(), Environment.defaultEnvironment(), @@ -352,16 +371,25 @@ public class DynamicProvisioningMaintainerTest { return this; } + private MockHostProvisioner without(Behaviour first, Behaviour... rest) { + Set<Behaviour> behaviours = new HashSet<>(this.behaviours); + behaviours.removeAll(EnumSet.of(first, rest)); + this.behaviours = behaviours.isEmpty() ? EnumSet.noneOf(Behaviour.class) : EnumSet.copyOf(behaviours); + return this; + } + private Node withIpAssigned(Node node) { if (node.parentHostname().isPresent()) return node; int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); Set<String> addresses = Set.of("::" + hostIndex + ":0"); - nameResolver.addRecord(node.hostname(), addresses.iterator().next()); Set<String> pool = new HashSet<>(); - for (int i = 1; i <= 2; i++) { - String ip = "::" + hostIndex + ":" + i; - pool.add(ip); - nameResolver.addRecord(node.hostname() + "-" + i, ip); + if (!behaviours.contains(Behaviour.failDnsUpdate)) { + nameResolver.addRecord(node.hostname(), addresses.iterator().next()); + for (int i = 1; i <= 2; i++) { + String ip = "::" + hostIndex + ":" + i; + pool.add(ip); + nameResolver.addRecord(node.hostname() + "-" + i, ip); + } } return node.with(node.ipConfig().with(addresses).with(IP.Pool.of(pool))); } @@ -369,6 +397,7 @@ public class DynamicProvisioningMaintainerTest { enum Behaviour { failProvisioning, failDeprovisioning, + failDnsUpdate, } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index 0a78ead5ca9..58a8edb4631 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; @@ -20,19 +19,16 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Report; import com.yahoo.vespa.hosted.provision.node.Reports; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.time.Duration; @@ -40,7 +36,6 @@ import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -239,8 +234,8 @@ public class FailedExpirerTest { public static final NodeResources defaultFlavor = new NodeResources(2, 8, 100, 2); public static final NodeResources dockerFlavor = new NodeResources(1, 4, 50, 1); - private final MockCurator curator = new MockCurator(); - private final ManualClock clock = new ManualClock(); + private final Curator curator; + private final ManualClock clock; private final ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("default")); @@ -248,20 +243,17 @@ public class FailedExpirerTest { private final NodeRepository nodeRepository; private final NodeRepositoryProvisioner provisioner; private final FailedExpirer expirer; + private final ProvisioningTester tester; public FailureScenario(SystemName system, Environment environment) { Zone zone = new Zone(system, environment, RegionName.defaultName()); - this.nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("registry.example.com/docker-image"), - new InMemoryFlagSource(), - true, - 0, 1000); - this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); + this.tester = new ProvisioningTester.Builder().zone(zone) + .flavors(nodeFlavors.getFlavors()) + .build(); + this.curator = tester.getCurator(); + this.clock = tester.clock(); + this.nodeRepository = tester.nodeRepository(); + this.provisioner = tester.provisioner(); this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30), new TestMetric()); } @@ -278,13 +270,12 @@ public class FailedExpirerTest { .orElseThrow(() -> new IllegalArgumentException("No such node: " + hostname)); } - public FailureScenario withNode(NodeType type, NodeResources flavor, String hostname, String parentHostname) { - nodeRepository.addNodes(List.of(nodeRepository.createNode(UUID.randomUUID().toString(), - hostname, - Optional.ofNullable(parentHostname), - new Flavor(flavor), - type)), - Agent.system); + public FailureScenario withNode(NodeType type, NodeResources resources, String hostname, String parentHostname) { + if (parentHostname != null) { + tester.makeReadyVirtualNodes(1, 0, resources, Optional.of(parentHostname), (index) -> hostname); + } else { + tester.makeProvisionedNodes(1, (index) -> hostname, new Flavor(resources), Optional.empty(), type, 0, false); + } return this; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java deleted file mode 100644 index 106d1b11a13..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * Generic maintenance tester - * - * @author bratseth - */ -public class MaintenanceTester { - - private final MockCurator curator = new MockCurator(); - public final ManualClock clock = new ManualClock(Instant.ofEpochMilli(0L)); // determinism - private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - public final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - - public MaintenanceTester() { - curator.setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); - } - - public NodeRepository nodeRepository() { return nodeRepository; } - - public void createReadyTenantNodes(int count) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodes = simulateInitialReboot(nodes); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - public void createReadyHostNodes(int count) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodes = simulateInitialReboot(nodes); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - /** Simulate the initial reboot the node performs when it's in dirty */ - private List<Node> simulateInitialReboot(List<Node> nodes) { - return nodes.stream() - .map(n -> n.withCurrentRebootGeneration(n.status().reboot().wanted(), Instant.now(clock))) - .collect(Collectors.toList()); - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 6ae5e667134..a7c04636662 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; @@ -80,21 +81,10 @@ public class MetricsReporterTest { @Test public void test_registered_metric() { NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - Curator curator = new MockCurator(); - NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - Clock.systemUTC(), - Zone.defaultZone(), - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant); - nodeRepository.addNodes(List.of(node), Agent.system); - Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy); - nodeRepository.addNodes(List.of(hostNode), Agent.system); + ProvisioningTester tester = new ProvisioningTester.Builder().flavors(nodeFlavors.getFlavors()).build(); + NodeRepository nodeRepository = tester.nodeRepository(); + tester.makeProvisionedNodes(1, "default", NodeType.tenant, 0); + tester.makeProvisionedNodes(1, "default", NodeType.proxy, 0); Map<String, Number> expectedMetrics = new TreeMap<>(); expectedMetrics.put("hostedVespa.provisionedHosts", 1); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index 1c044a8dfa6..5117a7b7397 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -5,31 +5,24 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub; import com.yahoo.vespa.hosted.provision.testutils.TestHostLivenessTracker; @@ -57,12 +50,12 @@ public class NodeFailTester { public static final ApplicationId app1 = ApplicationId.from("foo1", "bar", "fuz"); public static final ApplicationId app2 = ApplicationId.from("foo2", "bar", "fuz"); public static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "docker"); - private static final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); private static final Duration downtimeLimitOneHour = Duration.ofMinutes(60); // Components with state public final ManualClock clock; public final NodeRepository nodeRepository; + public final ProvisioningTester tester; public NodeFailer failer; public ServiceMonitorStub serviceMonitor; public MockDeployer deployer; @@ -73,21 +66,13 @@ public class NodeFailTester { private final Curator curator; private NodeFailTester() { - clock = new ManualClock(); - curator = new MockCurator(); - nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); - hostLivenessTracker = new TestHostLivenessTracker(clock); orchestrator = new OrchestratorMock(); + tester = new ProvisioningTester.Builder().orchestrator(orchestrator).flavors(nodeFlavors.getFlavors()).build(); + clock = tester.clock(); + curator = tester.getCurator(); + nodeRepository = tester.nodeRepository(); + provisioner = tester.provisioner(); + hostLivenessTracker = new TestHostLivenessTracker(clock); } public static NodeFailTester withTwoApplications() { @@ -123,7 +108,7 @@ public class NodeFailTester { int nodesPerHost = 3; List<Node> hosts = tester.createHostNodes(numberOfHosts); for (int i = 0; i < hosts.size(); i++) { - tester.createReadyNodes(nodesPerHost, i * nodesPerHost, Optional.of("parent" + i), + tester.createReadyNodes(nodesPerHost, i * nodesPerHost, Optional.of("parent" + (i + 1)), new NodeResources(1, 4, 100, 0.3), NodeType.tenant); } @@ -246,7 +231,7 @@ public class NodeFailTester { private List<Node> createReadyNodes(int count, int startIndex, Optional<String> parentHostname, Flavor flavor, NodeType nodeType) { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < startIndex + count; i++) - nodes.add(nodeRepository.createNode("node" + i, "host" + i, parentHostname, flavor, nodeType)); + nodes.add(nodeRepository.createNode("node" + i, "host" + i, IP.Config.EMPTY, parentHostname, flavor, Optional.empty(), nodeType)); nodes = nodeRepository.addNodes(nodes, Agent.system); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); @@ -254,10 +239,9 @@ public class NodeFailTester { } private List<Node> createHostNodes(int count) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host)); - nodes = nodeRepository.addNodes(nodes, Agent.system); + List<Node> nodes = tester.makeProvisionedNodes(count, (index) -> "parent" + index, + nodeFlavors.getFlavorOrThrow("default"), + Optional.empty(), NodeType.host, 10, false); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java index 3ff81070516..ebcb4c3e0dc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java @@ -2,10 +2,14 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.component.Version; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.time.Duration; @@ -24,68 +28,74 @@ public class NodeRebooterTest { public void testRebootScheduling() { var rebootInterval = Duration.ofDays(30); var flagSource = new InMemoryFlagSource().withIntFlag(Flags.REBOOT_INTERVAL_IN_DAYS.id(), (int) rebootInterval.toDays()); - var tester = new MaintenanceTester(); - tester.createReadyHostNodes(15); - NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, flagSource, new TestMetric()); + var tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + ((MockCurator) tester.getCurator()).setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); - assertReadyHosts(15, tester, 0L); + makeReadyHosts(15, tester); + NodeRepository nodeRepository = tester.nodeRepository(); + NodeRebooter rebooter = new NodeRebooter(nodeRepository, tester.clock(), flagSource, new TestMetric()); + + assertReadyHosts(15, nodeRepository, 0L); // No reboots within 0x-1x reboot interval - tester.clock.advance(rebootInterval); + tester.clock().advance(rebootInterval); rebooter.maintain(); - simulateReboot(tester); - assertReadyHosts(15, tester, 0L); + simulateReboot(nodeRepository); + assertReadyHosts(15, nodeRepository, 0L); // All nodes/hosts reboots within 1x-2x reboot interval - tester.clock.advance(rebootInterval); + tester.clock().advance(rebootInterval); rebooter.maintain(); - simulateReboot(tester); - assertReadyHosts(15, tester, 1L); + simulateReboot(nodeRepository); + assertReadyHosts(15, nodeRepository, 1L); // OS upgrade just before reboots would have been scheduled again - tester.clock.advance(rebootInterval); - scheduleOsUpgrade(tester); - simulateOsUpgrade(tester); + tester.clock().advance(rebootInterval); + scheduleOsUpgrade(nodeRepository); + simulateOsUpgrade(nodeRepository); rebooter.maintain(); - simulateReboot(tester); - assertReadyHosts(15, tester, 1L); + simulateReboot(nodeRepository); + assertReadyHosts(15, nodeRepository, 1L); // OS upgrade counts as reboot, so within 0x-1x there is no reboots - tester.clock.advance(rebootInterval); + tester.clock().advance(rebootInterval); rebooter.maintain(); - simulateReboot(tester); - assertReadyHosts(15, tester, 1L); + simulateReboot(nodeRepository); + assertReadyHosts(15, nodeRepository, 1L); // OS upgrade counts as reboot, but within 1x-2x reboots are scheduled again - tester.clock.advance(rebootInterval); + tester.clock().advance(rebootInterval); rebooter.maintain(); - simulateReboot(tester); - assertReadyHosts(15, tester, 2L); + simulateReboot(nodeRepository); + assertReadyHosts(15, nodeRepository, 2L); } @Test public void testRebootScheduledEvenWithSmallProbability() { Duration rebootInterval = Duration.ofDays(30); var flagSource = new InMemoryFlagSource().withIntFlag(Flags.REBOOT_INTERVAL_IN_DAYS.id(), (int) rebootInterval.toDays()); - var tester = new MaintenanceTester(); - tester.createReadyHostNodes(2); - NodeRebooter rebooter = new NodeRebooter(tester.nodeRepository, tester.clock, flagSource, new TestMetric()); + var tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + ((MockCurator) tester.getCurator()).setZooKeeperEnsembleConnectionSpec("zk1.host:1,zk2.host:2,zk3.host:3"); + + makeReadyHosts(2, tester); + NodeRepository nodeRepository = tester.nodeRepository(); + NodeRebooter rebooter = new NodeRebooter(nodeRepository, tester.clock(), flagSource, new TestMetric()); - assertReadyHosts(2, tester, 0L); + assertReadyHosts(2, nodeRepository, 0L); // No reboots within 0x-1x reboot interval - tester.clock.advance(rebootInterval); + tester.clock().advance(rebootInterval); rebooter.maintain(); - simulateReboot(tester); - assertReadyHosts(2, tester, 0L); + simulateReboot(nodeRepository); + assertReadyHosts(2, nodeRepository, 0L); // Advancing just a little bit into the 1x-2x interval, there is a >0 probability of // rebooting a host. Run until all have been scheduled. - tester.clock.advance(Duration.ofMinutes(25)); + tester.clock().advance(Duration.ofMinutes(25)); for (int i = 0;; ++i) { rebooter.maintain(); - simulateReboot(tester); - List<Node> nodes = tester.nodeRepository.getNodes(NodeType.host, Node.State.ready); + simulateReboot(nodeRepository); + List<Node> nodes = nodeRepository.getNodes(NodeType.host, Node.State.ready); int count = withCurrentRebootGeneration(1L, nodes).size(); if (count == 2) { break; @@ -93,33 +103,37 @@ public class NodeRebooterTest { } } - private void assertReadyHosts(int expectedCount, MaintenanceTester tester, long generation) { - List<Node> nodes = tester.nodeRepository.getNodes(NodeType.host, Node.State.ready); + private void assertReadyHosts(int expectedCount, NodeRepository nodeRepository, long generation) { + List<Node> nodes = nodeRepository.getNodes(NodeType.host, Node.State.ready); assertEquals(expectedCount, withCurrentRebootGeneration(generation, nodes).size()); } + private void makeReadyHosts(int count, ProvisioningTester tester) { + List<Node> hosts = tester.makeReadyNodes(count, new NodeResources(64, 256, 1000, 10), NodeType.host, 10); + } + /** Set current reboot generation to the wanted reboot generation whenever it is larger (i.e record a reboot) */ - private void simulateReboot(MaintenanceTester tester) { - for (Node node : tester.nodeRepository.getNodes(Node.State.ready, Node.State.active)) { + private void simulateReboot(NodeRepository nodeRepository) { + for (Node node : nodeRepository.getNodes(Node.State.ready, Node.State.active)) { if (node.status().reboot().wanted() > node.status().reboot().current()) - tester.nodeRepository.write(node.withCurrentRebootGeneration(node.status().reboot().wanted(), - tester.clock.instant()), () -> {}); + nodeRepository.write(node.withCurrentRebootGeneration(node.status().reboot().wanted(), + nodeRepository.clock().instant()), () -> {}); } } /** Schedule OS upgrade for all host nodes */ - private void scheduleOsUpgrade(MaintenanceTester tester) { - tester.nodeRepository.osVersions().setTarget(NodeType.host, Version.fromString("7.0"), Optional.empty(), false); + private void scheduleOsUpgrade(NodeRepository nodeRepository) { + nodeRepository.osVersions().setTarget(NodeType.host, Version.fromString("7.0"), Optional.empty(), false); } /** Simulate completion of an OS upgrade */ - private void simulateOsUpgrade(MaintenanceTester tester) { - var wantedOsVersion = tester.nodeRepository.osVersions().targetFor(NodeType.host); + private void simulateOsUpgrade(NodeRepository nodeRepository) { + var wantedOsVersion = nodeRepository.osVersions().targetFor(NodeType.host); if (wantedOsVersion.isEmpty()) return; - for (Node node : tester.nodeRepository.getNodes(Node.State.ready, Node.State.active)) { + for (Node node : nodeRepository.getNodes(Node.State.ready, Node.State.active)) { if (wantedOsVersion.get().isAfter(node.status().osVersion().current().orElse(Version.emptyVersion))) - tester.nodeRepository.write(node.withCurrentOsVersion(wantedOsVersion.get(), tester.clock.instant()), - () -> {}); + nodeRepository.write(node.withCurrentOsVersion(wantedOsVersion.get(), nodeRepository.clock().instant()), + () -> {}); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java index 41a1b4bac38..9d58c30a2c6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java @@ -6,36 +6,20 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; import org.junit.Test; import java.time.Duration; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -44,31 +28,16 @@ import static org.junit.Assert.assertEquals; */ public class OperatorChangeApplicationMaintainerTest { - private static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - - private NodeRepository nodeRepository; - private Fixture fixture; - @Test public void test_application_maintenance() { - ManualClock clock = new ManualClock(); - Curator curator = new MockCurator(); - Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - this.nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - this.fixture = new Fixture(zone, nodeRepository); - - createReadyNodes(15, this.fixture.nodeResources, nodeRepository); - createHostNodes(2, nodeRepository, nodeFlavors); - createProxyNodes(2, nodeRepository, nodeFlavors); + NodeResources hostResources = new NodeResources(64, 128, 2000, 10); + Fixture fixture = new Fixture(); + ManualClock clock = fixture.tester.clock(); + NodeRepository nodeRepository = fixture.nodeRepository; + + fixture.tester.makeReadyNodes(15, fixture.nodeResources); + fixture.tester.makeReadyNodes(2, hostResources); + fixture.tester.makeReadyNodes(2, fixture.nodeResources, NodeType.proxy); // Create applications fixture.activate(); @@ -104,39 +73,9 @@ public class OperatorChangeApplicationMaintainerTest { assertEquals("No further operator changes -> no (new) redeployments", 5, fixture.deployer.redeployments); } - private void createReadyNodes(int count, NodeResources resources, NodeRepository nodeRepository) { - createReadyNodes(count, new Flavor(resources), nodeRepository); - } - - private void createReadyNodes(int count, Flavor flavor, NodeRepository nodeRepository) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), flavor, NodeType.tenant)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - private void createHostNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - private void createProxyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("proxyNode" + i, "proxyHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - private static class Fixture { + final ProvisioningTester tester; final NodeRepository nodeRepository; final MockDeployer deployer; @@ -151,18 +90,14 @@ public class OperatorChangeApplicationMaintainerTest { final int wantedNodesApp2 = 7; final int wantedNodesApp3 = 2; - Fixture(Zone zone, NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, - zone, - new MockProvisionServiceProvider(), - new InMemoryFlagSource()); - + Fixture() { + this.tester = new ProvisioningTester.Builder().build(); + this.nodeRepository = tester.nodeRepository(); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.from(new ClusterResources(wantedNodesApp1, 1, nodeResources))), app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.from(new ClusterResources(wantedNodesApp2, 1, nodeResources))), app3, new MockDeployer.ApplicationContext(app3, clusterApp3, Capacity.fromRequiredNodeType(NodeType.proxy))) ; - this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps); + this.deployer = new MockDeployer(tester.provisioner(), nodeRepository.clock(), apps); } void activate() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index b437b8f5577..55a183c8ec1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -7,40 +7,26 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -49,31 +35,18 @@ import static org.junit.Assert.assertEquals; */ public class PeriodicApplicationMaintainerTest { - private static final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - private NodeRepository nodeRepository; private Fixture fixture; private ManualClock clock; @Before public void before() { - Curator curator = new MockCurator(); - Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - this.clock = new ManualClock(); - this.nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - this.fixture = new Fixture(zone, nodeRepository); - - createReadyNodes(15, fixture.nodeResources, nodeRepository); - createHostNodes(2, nodeRepository, nodeFlavors); + this.fixture = new Fixture(); + this.nodeRepository = fixture.nodeRepository; + this.clock = fixture.tester.clock(); + + fixture.tester.makeReadyNodes(15, fixture.nodeResources); + fixture.tester.makeReadyHosts(2, new NodeResources(64, 256, 2000, 10)); } @After @@ -229,32 +202,11 @@ public class PeriodicApplicationMaintainerTest { } } - private void createReadyNodes(int count, NodeResources nodeResources, NodeRepository nodeRepository) { - createReadyNodes(count, new Flavor(nodeResources), nodeRepository); - } - - private void createReadyNodes(int count, Flavor flavor, NodeRepository nodeRepository) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), flavor, NodeType.tenant)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - private void createHostNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - private class Fixture { final NodeRepository nodeRepository; final MockDeployer deployer; + final ProvisioningTester tester; final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -266,17 +218,13 @@ public class PeriodicApplicationMaintainerTest { private final TestablePeriodicApplicationMaintainer maintainer; - Fixture(Zone zone, NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, - zone, - new MockProvisionServiceProvider(), - new InMemoryFlagSource()); - + Fixture() { + this.tester = new ProvisioningTester.Builder().build(); + this.nodeRepository = tester.nodeRepository(); Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.from(new ClusterResources(wantedNodesApp1, 1, nodeResources))), app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.from(new ClusterResources(wantedNodesApp2, 1, nodeResources)))); - this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps); + this.deployer = new MockDeployer(tester.provisioner(), nodeRepository.clock(), apps); this.maintainer = new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofDays(1), // Long duration to prevent scheduled runs during test Duration.ofMinutes(30)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index e2ac644a419..12528a554d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -5,31 +5,18 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import org.junit.Test; import java.time.Duration; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -39,37 +26,23 @@ import static org.junit.Assert.assertFalse; */ public class ReservationExpirerTest { - private final Curator curator = new MockCurator(); - @Test public void ensure_reservation_times_out() { - ManualClock clock = new ManualClock(); NodeFlavors flavors = FlavorConfigBuilder.createDummies("default"); - NodeRepository nodeRepository = new NodeRepository(flavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - Zone.defaultZone(), - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new MockProvisionServiceProvider(), new InMemoryFlagSource()); + ProvisioningTester tester = new ProvisioningTester.Builder().flavors(flavors.getFlavors()).build(); + ManualClock clock = tester.clock(); + NodeRepository nodeRepository = tester.nodeRepository(); - List<Node> nodes = new ArrayList<>(2); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Flavor(new NodeResources(2, 8, 50, 1)), NodeType.tenant)); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Flavor(new NodeResources(2, 8, 50, 1)), NodeType.tenant)); - nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.host)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); + NodeResources nodeResources = new NodeResources(2, 8, 50, 1); + NodeResources hostResources = nodeResources.add(nodeResources).add(nodeResources); + tester.makeReadyNodes(2, nodeResources); + tester.makeReadyHosts(1, hostResources); // Reserve 2 nodes - assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.dirty).size()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); + assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.ready).size()); ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); - provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null); + tester.provisioner().prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources)), null); assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size()); // Reservation times out diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 9c5e74ae63c..aba5810784b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -7,42 +7,22 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; -import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider; -import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; -import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; -import com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import org.junit.Before; import org.junit.Test; import java.time.Duration; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -58,23 +38,13 @@ import static org.mockito.Mockito.verify; */ public class RetiredExpirerTest { + private final NodeResources hostResources = new NodeResources(64, 128, 2000, 10); private final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); - private Curator curator = new MockCurator(); - private final ManualClock clock = new ManualClock(); - private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); - private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default"); - private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, - new EmptyProvisionServiceProvider(), - curator, - clock, - zone, - new MockNameResolver().mockAnyLookup(), - DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), - new InMemoryFlagSource(), - true, - 0, 1000); - private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource()); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ManualClock clock = tester.clock(); + private final NodeRepository nodeRepository = tester.nodeRepository(); + private final NodeRepositoryProvisioner provisioner = tester.provisioner(); private final Orchestrator orchestrator = mock(Orchestrator.class); private static final Duration RETIRED_EXPIRATION = Duration.ofHours(12); @@ -87,8 +57,8 @@ public class RetiredExpirerTest { @Test public void ensure_retired_nodes_time_out() { - createReadyNodes(7, nodeResources, nodeRepository); - createHostNodes(4, nodeRepository, nodeFlavors); + tester.makeReadyNodes(7, nodeResources); + tester.makeReadyHosts(4, hostResources); ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -96,9 +66,9 @@ public class RetiredExpirerTest { // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); int wantedNodes; - activate(applicationId, cluster, wantedNodes=7, 1, provisioner); - activate(applicationId, cluster, wantedNodes=2, 1, provisioner); - activate(applicationId, cluster, wantedNodes=3, 1, provisioner); + activate(applicationId, cluster, wantedNodes=7, 1); + activate(applicationId, cluster, wantedNodes=2, 1); + activate(applicationId, cluster, wantedNodes=3, 1); assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.active).size()); assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); @@ -122,8 +92,8 @@ public class RetiredExpirerTest { @Test public void ensure_early_inactivation() throws OrchestrationException { - createReadyNodes(7, nodeResources, nodeRepository); - createHostNodes(4, nodeRepository, nodeFlavors); + tester.makeReadyNodes(7, nodeResources); + tester.makeReadyHosts(4, hostResources); ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz")); @@ -131,9 +101,9 @@ public class RetiredExpirerTest { // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); int wantedNodes; - activate(applicationId, cluster, wantedNodes=7, 1, provisioner); - activate(applicationId, cluster, wantedNodes=2, 1, provisioner); - activate(applicationId, cluster, wantedNodes=3, 1, provisioner); + activate(applicationId, cluster, wantedNodes=7, 1); + activate(applicationId, cluster, wantedNodes=2, 1); + activate(applicationId, cluster, wantedNodes=3, 1); assertEquals(7, nodeRepository.getNodes(applicationId, Node.State.active).size()); assertEquals(0, nodeRepository.getNodes(applicationId, Node.State.inactive).size()); @@ -180,39 +150,9 @@ public class RetiredExpirerTest { assertFalse(node.allocation().get().membership().retired()); } - private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) { - List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(nodes, groups, nodeResources)), null); - try (var lock = provisioner.lock(applicationId)) { - NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator)); - provisioner.activate(transaction, hosts, lock); - transaction.commit(); - } - } - - private void createReadyNodes(int count, NodeResources nodeResources, NodeRepository nodeRepository) { - createReadyNodes(count, new Flavor(nodeResources), nodeRepository); - } - - private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { - createReadyNodes(count, nodeFlavors.getFlavorOrThrow("default"), nodeRepository); - } - - private void createReadyNodes(int count, Flavor flavor, NodeRepository nodeRepository) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), flavor, NodeType.tenant)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - } - - private void createHostNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) { - List<Node> nodes = new ArrayList<>(count); - for (int i = 0; i < count; i++) - nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host)); - nodes = nodeRepository.addNodes(nodes, Agent.system); - nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); - nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); + private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups) { + Capacity capacity = Capacity.from(new ClusterResources(nodes, groups, nodeResources)); + tester.activate(applicationId, tester.prepare(applicationId, cluster, capacity)); } private RetiredExpirer createRetiredExpirer(Deployer deployer) { @@ -225,4 +165,5 @@ public class RetiredExpirerTest { Duration.ofDays(30), /* Maintenance interval, use large value so it never runs by itself */ RETIRED_EXPIRATION); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 62c6c0c9426..61274d04fe0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -29,7 +29,7 @@ public class AclProvisioningTest { private final NodeResources nodeResources = new NodeResources(2, 8, 50, 1); - private ProvisioningTester tester = new ProvisioningTester.Builder().build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); @Test public void trusted_nodes_for_allocated_node() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 257a3ce43d2..2a43b9f44f6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -110,7 +110,8 @@ public class ProvisioningTester { DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), flagSource, true, - spareCount, 1000); + spareCount, + 1000); this.orchestrator = orchestrator; this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource); this.capacityPolicies = new CapacityPolicies(nodeRepository); @@ -383,6 +384,13 @@ public class ProvisioningTester { return makeProvisionedNodes(n, asFlavor(flavor, type), Optional.empty(), type, ipAddressPoolSize, dualStack); } public List<Node> makeProvisionedNodes(int n, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) { + return makeProvisionedNodes(n, (index) -> "host-" + index + ".yahoo.com", flavor, reservedTo, type, ipAddressPoolSize, dualStack); + } + + public List<Node> makeProvisionedNodes(int n, Function<Integer, String> nodeNamer, Flavor flavor, Optional<TenantName> reservedTo, NodeType type, int ipAddressPoolSize, boolean dualStack) { + if (ipAddressPoolSize == 0 && type == NodeType.host) { + ipAddressPoolSize = 1; // Tenant hosts must have at least one IP in their pool + } List<Node> nodes = new ArrayList<>(n); for (int i = 0; i < n; i++) { @@ -393,12 +401,12 @@ public class ProvisioningTester { // name resolver already contains the next host - if this is the case - bump the indices and move on String testIp = String.format("127.0.0.%d", nextIP); MockNameResolver nameResolver = (MockNameResolver)nodeRepository().nameResolver(); - if (nameResolver.getHostname(testIp).isPresent()) { + if (nameResolver.resolveHostname(testIp).isPresent()) { nextHost += 100; nextIP += 100; } - String hostname = String.format("host-%d.yahoo.com", nextHost); + String hostname = nodeNamer.apply(nextHost); String ipv4 = String.format("127.0.0.%d", nextIP); String ipv6 = String.format("::%d", nextIP); @@ -499,12 +507,6 @@ public class ProvisioningTester { i -> String.format("%s-%03d", dockerHostId, i)); } - /** Creates a single of virtual docker node on a single parent host */ - public List<Node> makeReadyVirtualDockerNode(int index, NodeResources resources, String dockerHostId) { - return makeReadyVirtualNodes(1, index, resources, Optional.of(dockerHostId), - i -> String.format("%s-%03d", dockerHostId, i)); - } - /** Creates a set of virtual nodes without a parent host */ public List<Node> makeReadyVirtualNodes(int n, NodeResources resources) { return makeReadyVirtualNodes(n, 0, resources, Optional.empty(), @@ -512,13 +514,13 @@ public class ProvisioningTester { } /** Creates a set of virtual nodes on a single parent host */ - private List<Node> makeReadyVirtualNodes(int count, int startIndex, NodeResources flavor, Optional<String> parentHostId, + public List<Node> makeReadyVirtualNodes(int count, int startIndex, NodeResources resources, Optional<String> parentHostname, Function<Integer, String> nodeNamer) { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { String hostname = nodeNamer.apply(i); - nodes.add(nodeRepository.createNode("openstack-id", hostname, parentHostId, - new Flavor(flavor), NodeType.tenant)); + nodes.add(nodeRepository.createNode("node-id", hostname, IP.Config.EMPTY, parentHostname, + new Flavor(resources), Optional.empty(), NodeType.tenant)); } nodes = nodeRepository.addNodes(nodes, Agent.system); nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); |