diff options
14 files changed, 245 insertions, 71 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java index e356ee06ac6..63ad6ae1237 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeType.java @@ -75,4 +75,16 @@ public enum NodeType { public boolean canRun(NodeType type) { return childNodeTypes.contains(type); } + + /** Returns the host type of this */ + public NodeType hostType() { + if (isHost()) return this; + for (NodeType nodeType : values()) { + if (nodeType.childNodeTypes.size() == 1 && nodeType.canRun(this)) { + return nodeType; + } + } + throw new IllegalArgumentException("No host of " + this + " exists"); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index c8763f7154e..68caea53517 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -103,9 +103,9 @@ public final class Node implements Nodelike { } if (type != NodeType.host && reservedTo.isPresent()) - throw new IllegalArgumentException("Only hosts can be reserved to a tenant"); + throw new IllegalArgumentException("Only tenant hosts can be reserved to a tenant"); - if (type != NodeType.host && exclusiveTo.isPresent()) + if (!type.isHost() && exclusiveTo.isPresent()) throw new IllegalArgumentException("Only hosts can be exclusive to an application"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 204d4eea1c4..2979940ee22 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -90,13 +90,13 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { /** Resume provisioning of already provisioned hosts and their children */ private void resumeProvisioning(NodeList nodes, Mutex lock) { - Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream() + Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant, NodeType.config).asList().stream() .filter(node -> node.parentHostname().isPresent()) .collect(Collectors.groupingBy( node -> node.parentHostname().get(), Collectors.toSet())); - nodes.state(Node.State.provisioned).hosts().forEach(host -> { + nodes.state(Node.State.provisioned).nodeType(NodeType.host, NodeType.confighost).forEach(host -> { Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); try { List<Node> updatedNodes = hostProvisioner.provision(host, children); @@ -197,10 +197,10 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toMap(Node::hostname, Function.identity()))); nodes.stream() - .filter(node -> node.allocation().isPresent()) - .flatMap(node -> node.parentHostname().stream()) - .distinct() - .forEach(hostsByHostname::remove); + .filter(node -> node.allocation().isPresent()) + .flatMap(node -> node.parentHostname().stream()) + .distinct() + .forEach(hostsByHostname::remove); return List.copyOf(hostsByHostname.values()); } @@ -246,8 +246,8 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private List<Node> provisionHosts(int count, NodeResources nodeResources) { try { Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); - List<Integer> provisionIndexes = nodeRepository().database().getProvisionIndexes(count); - List<Node> hosts = hostProvisioner.provisionHosts(provisionIndexes, nodeResources, + List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); + List<Node> hosts = hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, HostSharing.shared) .stream() .map(ProvisionedHost::generateHost) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index c2fe063dae6..3ed29e14527 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -533,15 +533,15 @@ public class CuratorDatabaseClient { .collect(Collectors.toUnmodifiableList()); } - /** Returns a given number of unique provision indexes */ - public List<Integer> getProvisionIndexes(int numIndexes) { - if (numIndexes < 1) - throw new IllegalArgumentException("numIndexes must be a positive integer, was " + numIndexes); - - int firstProvisionIndex = (int) provisionIndexCounter.add(numIndexes) - numIndexes; - return IntStream.range(0, numIndexes) - .mapToObj(i -> firstProvisionIndex + i) - .collect(Collectors.toList()); + /** Returns a given number of unique provision indices */ + public List<Integer> readProvisionIndices(int count) { + if (count < 1) + throw new IllegalArgumentException("count must be a positive integer, was " + count); + + int firstIndex = (int) provisionIndexCounter.add(count) - count; + return IntStream.range(0, count) + .mapToObj(i -> firstIndex + i) + .collect(Collectors.toList()); } public CacheStats cacheStats() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 0a22cc1cc58..e6473e62922 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; -import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -90,19 +89,23 @@ public class GroupPreparer { allocateOsRequirement); if (nodeRepository.zone().getCloud().dynamicProvisioning()) { + NodeType hostType = allocation.nodeType().hostType(); final Version osVersion; if (allocateOsRequirement.equals("rhel8")) { osVersion = new Version(8, Integer.MAX_VALUE /* always use latest 8 version */, 0); } else { - osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); + osVersion = nodeRepository.osVersions().targetFor(hostType).orElse(Version.emptyVersion); } - - List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() - .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), - deficit.getFlavor(), - application, - osVersion, - requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any)) + List<ProvisionedHost> provisionedHosts = allocation.nodeDeficit() + .map(deficit -> { + HostSharing sharing = requestedNodes.isExclusive() ? HostSharing.exclusive : HostSharing.any; + return hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.getCount()), + hostType, + deficit.getFlavor(), + application, + osVersion, + sharing); + }) .orElseGet(List::of); // At this point we have started provisioning of the hosts, the first priority is to make sure that diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index ae8c6757b5a..bfb526a518f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import java.util.List; @@ -32,8 +33,9 @@ public interface HostProvisioner { /** * Schedule provisioning of a given number of hosts. * - * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames + * @param provisionIndices list of unique provision indices which will be used to generate the node hostnames * on the form of <code>[prefix][index].[domain]</code> + * @param hostType The host type to provision * @param resources the resources needed per node - the provisioned host may be significantly larger * @param applicationId id of the application that will own the provisioned host * @param osVersion the OS version to use. If this version does not exist, implementations may choose a suitable @@ -41,8 +43,11 @@ public interface HostProvisioner { * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned. * @return list of {@link ProvisionedHost} describing the provisioned nodes */ - List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, - ApplicationId applicationId, Version osVersion, + List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, + NodeType hostType, + NodeResources resources, + ApplicationId applicationId, + Version osVersion, HostSharing sharing); /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 0eb933a7dcc..d96ce354337 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -281,7 +281,7 @@ class NodeAllocation { /** Returns true if the content of this list is sufficient to meet the request */ boolean fulfilled() { - return requestedNodes.fulfilledBy(accepted); + return requestedNodes.fulfilledBy(accepted()); } /** Returns true if this allocation was already fulfilled and resulted in no new changes */ @@ -290,17 +290,47 @@ class NodeAllocation { } /** - * Returns {@link FlavorCount} describing the docker node deficit for the given {@link NodeSpec}. + * Returns {@link FlavorCount} describing the node deficit for the given {@link NodeSpec}. * - * @return empty if the requested spec is not count based or the requested flavor type is not docker or - * the request is already fulfilled. Otherwise returns {@link FlavorCount} containing the required flavor - * and node count to cover the deficit. + * @return empty if the requested spec is already fulfilled. Otherwise returns {@link FlavorCount} containing the + * flavor and node count required to cover the deficit. */ - Optional<FlavorCount> getFulfilledDockerDeficit() { - return Optional.of(requestedNodes) - .filter(NodeSpec.CountNodeSpec.class::isInstance) - .map(spec -> new FlavorCount(spec.resources().get(), spec.fulfilledDeficitCount(accepted))) - .filter(flavorCount -> flavorCount.getCount() > 0); + Optional<FlavorCount> nodeDeficit() { + if (nodeType() != NodeType.config && nodeType() != NodeType.tenant) { + return Optional.empty(); // Requests for these node types never have a deficit + } + return Optional.of(new FlavorCount(requestedNodes.resources().orElseGet(NodeResources::unspecified), + requestedNodes.fulfilledDeficitCount(accepted()))) + .filter(flavorCount -> flavorCount.getCount() > 0); + } + + /** Returns the indices to use when provisioning hosts for this */ + List<Integer> provisionIndices(int count) { + if (count == 0) return List.of(); + NodeType hostType = requestedNodes.type().hostType(); + + // Tenant hosts have a continuously increasing index + if (hostType == NodeType.host) return nodeRepository.database().readProvisionIndices(count); + + // Infrastructure hosts have fixed indices: Their cluster index + 1 + int offset = 1; + Set<Integer> currentIndices = allNodes.nodeType(hostType) + .stream() + .map(node -> node.allocation().get().membership().index()) + .map(index -> index + offset) + .collect(Collectors.toSet()); + List<Integer> indices = new ArrayList<>(count); + for (int i = offset; indices.size() < count; i++) { + if (!currentIndices.contains(i)) { + indices.add(i); + } + } + return indices; + } + + /** The node type this is allocating */ + NodeType nodeType() { + return requestedNodes.type(); } /** @@ -367,6 +397,14 @@ class NodeAllocation { .collect(Collectors.toList()); } + /** Returns the number of nodes accepted this far */ + private int accepted() { + if (nodeType() == NodeType.tenant) return accepted; + // Infrastructure nodes are always allocated by type. Count all nodes as accepted so that we never exceed + // the wanted number of nodes for the type. + return allNodes.nodeType(nodeType()).size(); + } + /** Prefer to retire nodes we want the least */ private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) { return candidates.stream().sorted(Comparator.reverseOrder()).collect(Collectors.toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index b37c7b92ea4..c3cb805499c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -170,6 +171,8 @@ public interface NodeSpec { /** A node spec specifying a node type. This will accept all nodes of this type. */ class TypeNodeSpec implements NodeSpec { + private static final Map<NodeType, Integer> WANTED_NODE_COUNT = Map.of(NodeType.config, 3); + private final NodeType type; public TypeNodeSpec(NodeType type) { @@ -204,14 +207,17 @@ public interface NodeSpec { @Override public int fulfilledDeficitCount(int count) { - return 0; + // If no wanted count is specified for this node type, then any count fulfills the deficit + return Math.max(0, WANTED_NODE_COUNT.getOrDefault(type, 0) - count); } @Override public NodeSpec fraction(int divisor) { return this; } @Override - public Optional<NodeResources> resources() { return Optional.empty(); } + public Optional<NodeResources> resources() { + return Optional.empty(); + } @Override public boolean needsResize(Node node) { return false; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index 6ff4d4ca5f8..caaea1167b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -27,20 +27,23 @@ public class ProvisionedHost { private final String id; private final String hostHostname; private final Flavor hostFlavor; + private final NodeType hostType; private final Optional<ApplicationId> exclusiveTo; private final List<Address> nodeAddresses; private final NodeResources nodeResources; private final Version osVersion; - public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, Optional<ApplicationId> exclusiveTo, + public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, Optional<ApplicationId> exclusiveTo, List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion) { this.id = Objects.requireNonNull(id, "Host id must be set"); this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); + this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); this.exclusiveTo = Objects.requireNonNull(exclusiveTo, "exclusiveTo must be set"); this.nodeAddresses = validateNodeAddresses(nodeAddresses); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set"); + if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); } private static List<Address> validateNodeAddresses(List<Address> nodeAddresses) { @@ -54,7 +57,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { Node.Builder builder = Node - .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, NodeType.host) + .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, hostType) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); exclusiveTo.ifPresent(builder::exclusiveTo); return builder.build(); @@ -62,7 +65,7 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the node running on this physical host */ public Node generateNode() { - return Node.reserve(Set.of(), nodeHostname(), hostHostname, nodeResources, NodeType.tenant).build(); + return Node.reserve(Set.of(), nodeHostname(), hostHostname, nodeResources, hostType.childNodeType()).build(); } public String getId() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 4c9323ca68e..25e74df677b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Address; @@ -52,18 +53,20 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, + public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, NodeType hostType, NodeResources resources, ApplicationId applicationId, Version osVersion, HostSharing sharing) { Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream().filter(f -> compatible(f, resources)) .findFirst() .orElseThrow(() -> new OutOfCapacityException("No host flavor matches " + resources))); List<ProvisionedHost> hosts = new ArrayList<>(); - for (int index : provisionIndexes) { - hosts.add(new ProvisionedHost("host" + index, - "hostname" + index, + for (int index : provisionIndices) { + String hostHostname = hostType == NodeType.host ? "hostname" + index : hostType.name() + index; + hosts.add(new ProvisionedHost("id-of-" + hostType.name() + index, + hostHostname, hostFlavor, + hostType, Optional.empty(), - createAddressesForHost(hostFlavor, index), + createAddressesForHost(hostType, hostFlavor, index), resources, osVersion)); } @@ -132,15 +135,22 @@ public class MockHostProvisioner implements HostProvisioner { return flavor.resources().compatibleWith(resourcesToVerify); } - private List<Address> createAddressesForHost(Flavor flavor, int hostIndex) { + private List<Address> createAddressesForHost(NodeType hostType, Flavor flavor, int hostIndex) { long numAddresses = Math.max(1, Math.round(flavor.resources().bandwidthGbps())); return IntStream.range(0, (int) numAddresses) - .mapToObj(i -> new Address("nodename" + hostIndex + "_" + i)) + .mapToObj(i -> { + String hostname = hostType == NodeType.host + ? "nodename" + hostIndex + "_" + i + : hostType.childNodeType().name() + i; + return new Address(hostname); + }) .collect(Collectors.toList()); } private Node withIpAssigned(Node node) { - if (node.parentHostname().isPresent()) return node; + if (!node.type().isHost()) { + return node.with(node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname()))); + } int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); Set<String> addresses = Set.of("::" + hostIndex + ":0"); Set<String> ipAddressPool = new HashSet<>(); @@ -152,7 +162,6 @@ public class MockHostProvisioner implements HostProvisioner { nameResolver.addRecord(node.hostname() + "-" + i, ip); } } - IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool); return node.with(node.ipConfig().withPrimary(addresses).withPool(pool)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 43a7be81c88..4a1b3df0514 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -12,6 +12,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -28,6 +29,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; @@ -37,6 +39,7 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.Set; @@ -46,7 +49,9 @@ import java.util.stream.Stream; import static com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner.Behaviour; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author freva @@ -414,6 +419,93 @@ public class DynamicProvisioningMaintainerTest { assertCfghost3IsDeprovisioned(tester); } + @Test + public void replace_config_server() { + Cloud cloud = Cloud.builder().dynamicProvisioning(true).build(); + DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup()); + ProvisioningTester tester = dynamicProvisioningTester.provisioningTester; + dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default"); + + // Initial config server hosts are provisioned manually + ApplicationId hostApp = ApplicationId.from("hosted-vespa", "configserver-host", "default"); + List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", NodeType.confighost).stream() + .sorted(Comparator.comparing(Node::hostname)) + .collect(Collectors.toList()); + tester.prepareAndActivateInfraApplication(hostApp, NodeType.confighost); + + // Provision config servers + ApplicationId configSrvApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); + for (int i = 0; i < provisionedHosts.size(); i++) { + tester.makeReadyChildren(1, i + 1, NodeResources.unspecified(), NodeType.config, + provisionedHosts.get(i).hostname(), (nodeIndex) -> "cfg" + nodeIndex); + } + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + + // Expected number of hosts and children are provisioned + NodeList allNodes = tester.nodeRepository().nodes().list(); + NodeList configHosts = allNodes.nodeType(NodeType.confighost); + NodeList configNodes = allNodes.nodeType(NodeType.config); + assertEquals(3, configHosts.size()); + assertEquals(3, configNodes.size()); + String hostnameToRemove = provisionedHosts.get(1).hostname(); + Supplier<Node> hostToRemove = () -> tester.nodeRepository().nodes().node(hostnameToRemove).get(); + Supplier<Node> nodeToRemove = () -> tester.nodeRepository().nodes().node(configNodes.childrenOf(hostnameToRemove).first().get().hostname()).get(); + + // Retire and deprovision host + tester.nodeRepository().nodes().deprovision(hostToRemove.get(), Agent.system, tester.clock().instant()); + tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.system, getClass().getSimpleName()); + assertSame("Host moves to parked", Node.State.parked, hostToRemove.get().state()); + assertSame("Node remains active", Node.State.active, nodeToRemove.get().state()); + assertTrue("Node wants to retire", nodeToRemove.get().status().wantToRetire()); + + // Redeployment of config server application retires node + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + assertTrue("Redeployment retires node", nodeToRemove.get().allocation().get().membership().retired()); + + // Config server becomes removable (done by RetiredExpirer in a real system) and redeployment moves it + // to inactive + tester.nodeRepository().nodes().setRemovable(configSrvApp, List.of(nodeToRemove.get())); + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + assertEquals("Node moves to inactive", Node.State.inactive, nodeToRemove.get().state()); + + // Node is completely removed (done by InactiveExpirer and host-admin in a real system) + Node inactiveConfigServer = nodeToRemove.get(); + int removedIndex = inactiveConfigServer.allocation().get().membership().index(); + tester.nodeRepository().nodes().removeRecursively(inactiveConfigServer, true); + assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.config).size()); + + // Host is removed + dynamicProvisioningTester.maintainer.maintain(); + assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.confighost).size()); + + // Next deployment starts provisioning a new host and child + try { + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + fail("Expected provisioning to fail"); + } catch (ParentHostUnavailableException ignored) {} + Node newNode = tester.nodeRepository().nodes().list(Node.State.reserved).nodeType(NodeType.config).first().get(); + + // Resume provisioning and activate host + dynamicProvisioningTester.maintainer.maintain(); + List<ProvisionedHost> newHosts = dynamicProvisioningTester.hostProvisioner.provisionedHosts(); + assertEquals(1, newHosts.size()); + tester.nodeRepository().nodes().setReady(newHosts.get(0).hostHostname(), Agent.operator, getClass().getSimpleName()); + tester.prepareAndActivateInfraApplication(hostApp, NodeType.confighost); + assertEquals(3, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); + + // Redeployment of config server app actives new node + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + newNode = tester.nodeRepository().nodes().node(newNode.hostname()).get(); + assertSame(Node.State.active, newNode.state()); + assertEquals("Removed index is reused", removedIndex, newNode.allocation().get().membership().index()); + + // Next redeployment does nothing + NodeList nodesBefore = tester.nodeRepository().nodes().list().nodeType(NodeType.config); + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + NodeList nodesAfter = tester.nodeRepository().nodes().list().nodeType(NodeType.config); + assertEquals(nodesBefore, nodesAfter); + } + private void assertCfghost3IsActive(DynamicProvisioningTester tester) { assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size()); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); @@ -444,11 +536,10 @@ public class DynamicProvisioningMaintainerTest { private final ProvisioningTester provisioningTester; public DynamicProvisioningTester() { - this(Cloud.builder().dynamicProvisioning(true).build()); + this(Cloud.builder().dynamicProvisioning(true).build(), new MockNameResolver()); } - public DynamicProvisioningTester(Cloud cloud) { - MockNameResolver nameResolver = new MockNameResolver(); + public DynamicProvisioningTester(Cloud cloud, MockNameResolver nameResolver) { this.hostProvisioner = new MockHostProvisioner(flavors.getFlavors(), nameResolver, 0); this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(), Environment.defaultEnvironment(), @@ -536,3 +627,4 @@ public class DynamicProvisioningMaintainerTest { } } + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 3b8e7075488..4db1b86419b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -71,7 +71,7 @@ public class DynamicDockerProvisionTest { mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); - verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), resources, application1, + verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), NodeType.host, resources, application1, Version.emptyVersion, HostSharing.any); // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes @@ -97,7 +97,7 @@ public class DynamicDockerProvisionTest { ApplicationId application3 = ProvisioningTester.applicationId(); mockHostProvisioner(hostProvisioner, "large", 3, application3); prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, resources); - verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), resources, application3, + verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), NodeType.host, resources, application3, Version.emptyVersion, HostSharing.exclusive); // Total of 20 nodes should now be in node-repo, 8 active hosts and 12 active nodes @@ -427,7 +427,7 @@ public class DynamicDockerProvisionTest { doAnswer(invocation -> { Flavor hostFlavor = tester.nodeRepository().flavors().getFlavorOrThrow(hostFlavorName); List<Integer> provisionIndexes = (List<Integer>) invocation.getArguments()[0]; - NodeResources nodeResources = (NodeResources) invocation.getArguments()[1]; + NodeResources nodeResources = (NodeResources) invocation.getArguments()[2]; return provisionIndexes.stream() .map(hostIndex -> { @@ -449,7 +449,7 @@ public class DynamicDockerProvisionTest { return provisionedHost; }) .collect(Collectors.toList()); - }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any()); + }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any(), any()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 0986f2954a7..c269b4642ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -33,7 +33,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -1017,7 +1016,7 @@ public class ProvisioningTest { private Set<HostSpec> prepare(ApplicationId application, ProvisioningTester tester, ClusterSpec cluster, int nodeCount, int groups, boolean required, NodeResources nodeResources) { - if (nodeCount == 0) return Collections.emptySet(); // this is a shady practice + if (nodeCount == 0) return Set.of(); // this is a shady practice return new HashSet<>(tester.prepare(application, cluster, nodeCount, groups, required, nodeResources)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index c0306215f6d..eefbd03ce4e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -513,17 +513,18 @@ public class ProvisioningTester { index -> UUID.randomUUID().toString()); } - /** Creates a set of virtual nodes on a single parent host */ - public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, String parentHostname, - Function<Integer, String> nodeNamer) { + /** Create one or more child nodes on given parent host */ + public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, NodeType nodeType, + String parentHostname, Function<Integer, String> nodeNamer) { + if (nodeType.isHost()) throw new IllegalArgumentException("Non-child node type: " + nodeType); List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { String hostname = nodeNamer.apply(i); IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), Set.of()); - - Node.Builder builder = Node.create("node-id", ipConfig, hostname, new Flavor(resources), NodeType.tenant); - builder.parentHostname(parentHostname); - nodes.add(builder.build()); + Node node = Node.create("node-id", ipConfig, hostname, new Flavor(resources), nodeType) + .parentHostname(parentHostname) + .build(); + nodes.add(node); } nodes = nodeRepository.nodes().addNodes(nodes, Agent.system); nodes = nodeRepository.nodes().deallocate(nodes, Agent.system, getClass().getSimpleName()); @@ -531,6 +532,12 @@ public class ProvisioningTester { return nodes; } + /** Create one or more child nodes on given parent host */ + public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, String parentHostname, + Function<Integer, String> nodeNamer) { + return makeReadyChildren(count, startIndex, resources, NodeType.tenant, parentHostname, nodeNamer); + } + public void activateTenantHosts() { prepareAndActivateInfraApplication(applicationId(), NodeType.host); } |