diff options
Diffstat (limited to 'node-repository/src/test/java/com')
6 files changed, 181 insertions, 246 deletions
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 0619b0ad645..77c3a5209e2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.collections.Pair; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; @@ -21,17 +20,12 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; -import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -39,6 +33,9 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * @author bratseth + */ class AutoscalingTester { private final ProvisioningTester provisioningTester; @@ -296,45 +293,14 @@ class AutoscalingTester { } - private class MockHostProvisioner implements HostProvisioner { + private class MockHostProvisioner extends com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner { - private final List<Flavor> hostFlavors; - - public MockHostProvisioner(List<Flavor> hostFlavors) { - this.hostFlavors = hostFlavors; - } - - @Override - public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, - ApplicationId applicationId, Version osVersion, - HostSharing sharing) { - Flavor hostFlavor = hostFlavors.stream().filter(f -> matches(f, resources)).findAny() - .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors)); - - List<ProvisionedHost> hosts = new ArrayList<>(); - for (int index : provisionIndexes) { - hosts.add(new ProvisionedHost("host" + index, - "hostname" + index, - hostFlavor, - Optional.empty(), - List.of(new Address("nodename" + index)), - resources, - osVersion)); - } - return hosts; + public MockHostProvisioner(List<Flavor> flavors) { + super(flavors); } @Override - public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { - throw new RuntimeException("Not implemented"); - } - - @Override - public void deprovision(Node host) { - throw new RuntimeException("Not implemented"); - } - - private boolean matches(Flavor flavor, NodeResources resources) { + public boolean compatible(Flavor flavor, NodeResources resources) { NodeResources flavorResources = hostResourcesCalculator.advertisedResourcesOf(flavor); if (flavorResources.storageType() == NodeResources.StorageType.remote && resources.diskGb() <= flavorResources.diskGb()) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 56cf8d02149..f47bcd0d550 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -12,9 +12,11 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.ClusterCapacity; @@ -27,11 +29,10 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.IP; -import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; -import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.duper.ConfigServerHostApplication; @@ -39,21 +40,19 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashSet; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.MockHostProvisioner.Behaviour; +import static com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner.Behaviour; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author freva @@ -115,7 +114,7 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); assertTrue("Failed host is deprovisioned", tester.nodeRepository.nodes().node(failedHost.get().hostname()).isEmpty()); - assertEquals(1, tester.hostProvisioner.deprovisionedHosts); + assertEquals(1, tester.hostProvisioner.deprovisionedHosts()); } @Test @@ -126,7 +125,7 @@ public class DynamicProvisioningMaintainerTest { new ClusterCapacity(1, 16, 24, 100, 1.0)), ClusterCapacity.class); - assertEquals(0, tester.hostProvisioner.provisionedHosts.size()); + assertEquals(0, tester.hostProvisioner.provisionedHosts().size()); assertEquals(11, tester.nodeRepository.nodes().list().size()); assertTrue(tester.nodeRepository.nodes().node("host2").isPresent()); assertTrue(tester.nodeRepository.nodes().node("host2-1").isPresent()); @@ -136,7 +135,7 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); - assertEquals(2, tester.hostProvisioner.provisionedHosts.size()); + assertEquals(2, tester.hostProvisioner.provisionedHosts().size()); assertEquals(2, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10))); NodeList nodesAfter = tester.nodeRepository.nodes().list(); assertEquals(11, nodesAfter.size()); // 2 removed, 2 added @@ -151,13 +150,13 @@ public class DynamicProvisioningMaintainerTest { public void preprovision_with_shared_host() { var tester = new DynamicProvisioningTester().addInitialNodes(); // Makes provisioned hosts 48-128-1000-10 - tester.hostProvisioner.provisionSharedHost("host4"); + tester.hostProvisioner.overrideHostFlavor("host4"); tester.flagSource.withListFlag(PermanentFlags.PREPROVISION_CAPACITY.id(), List.of(new ClusterCapacity(2, 1, 30, 20, 3.0)), ClusterCapacity.class); - assertEquals(0, tester.hostProvisioner.provisionedHosts.size()); + assertEquals(0, tester.hostProvisioner.provisionedHosts().size()); assertEquals(11, tester.nodeRepository.nodes().list().size()); assertTrue(tester.nodeRepository.nodes().node("host2").isPresent()); assertTrue(tester.nodeRepository.nodes().node("host2-1").isPresent()); @@ -196,7 +195,7 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); - assertEquals(2, tester.hostProvisioner.provisionedHosts.size()); + assertEquals(2, tester.hostProvisioner.provisionedHosts().size()); assertEquals(2, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10))); assertEquals(10, tester.nodeRepository.nodes().list().size()); // 3 removed, 2 added assertTrue("preprovision capacity is prefered on shared hosts", tester.nodeRepository.nodes().node("host3").isEmpty()); @@ -212,7 +211,7 @@ public class DynamicProvisioningMaintainerTest { tester.maintainer.maintain(); assertEquals("one provisioned host has been deprovisioned, so there are 2 -> 1 provisioned hosts", - 1, tester.hostProvisioner.provisionedHosts.size()); + 1, tester.hostProvisioner.provisionedHosts().size()); assertEquals(1, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10))); assertEquals(9, tester.nodeRepository.nodes().list().size()); // 4 removed, 2 added if (tester.nodeRepository.nodes().node("hostname100").isPresent()) { @@ -226,7 +225,7 @@ public class DynamicProvisioningMaintainerTest { } private void verifyFirstMaintain(DynamicProvisioningTester tester) { - assertEquals(1, tester.hostProvisioner.provisionedHosts.size()); + assertEquals(1, tester.hostProvisioner.provisionedHosts().size()); assertEquals(1, tester.provisionedHostsMatching(new NodeResources(48, 128, 1000, 10))); assertEquals(10, tester.nodeRepository.nodes().list().size()); // 2 removed, 1 added assertTrue("Failed host 'host2' is deprovisioned", tester.nodeRepository.nodes().node("host2").isEmpty()); @@ -266,17 +265,17 @@ public class DynamicProvisioningMaintainerTest { private void assertWithMinCount(int minCount, int provisionCount, int deprovisionCount) { var tester = new DynamicProvisioningTester().addInitialNodes(); - tester.hostProvisioner.provisionSharedHost("host4"); + tester.hostProvisioner.overrideHostFlavor("host4"); tester.flagSource.withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(null, minCount), SharedHost.class); tester.maintainer.maintain(); - assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts.size()); - assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts); + assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts().size()); + assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts()); // Verify next maintain is a no-op tester.maintainer.maintain(); - assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts.size()); - assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts); + assertEquals(provisionCount, tester.hostProvisioner.provisionedHosts().size()); + assertEquals(deprovisionCount, tester.hostProvisioner.deprovisionedHosts()); } @Test @@ -300,14 +299,14 @@ public class DynamicProvisioningMaintainerTest { // Hosts are provisioned assertEquals(2, tester.provisionedHostsMatching(resources1)); - assertEquals(0, tester.hostProvisioner.deprovisionedHosts); + assertEquals(0, tester.hostProvisioner.deprovisionedHosts()); // Next maintenance run does nothing tester.assertNodesUnchanged(); // Pretend shared-host flag has been set to host4's flavor var sharedHostNodeResources = new NodeResources(48, 128, 1000, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote); - tester.hostProvisioner.provisionSharedHost("host4"); + tester.hostProvisioner.overrideHostFlavor("host4"); // Next maintenance run does nothing tester.assertNodesUnchanged(); @@ -421,6 +420,94 @@ public class DynamicProvisioningMaintainerTest { assertCfghost3IsDeprovisioned(tester); } + @Test + public void replace_config_server() { + Cloud cloud = Cloud.builder().dynamicProvisioning(true).build(); + DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup()); + ProvisioningTester tester = dynamicProvisioningTester.provisioningTester; + dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default"); + dynamicProvisioningTester.flagSource.withBooleanFlag(Flags.DYNAMIC_CONFIG_SERVER_PROVISIONING.id(), true); + + // Initial config server hosts are provisioned manually + ApplicationId hostApp = ApplicationId.from("hosted-vespa", "configserver-host", "default"); + List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", NodeType.confighost).stream() + .sorted(Comparator.comparing(Node::hostname)) + .collect(Collectors.toList()); + tester.prepareAndActivateInfraApplication(hostApp, NodeType.confighost); + + // Provision config servers + ApplicationId configSrvApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); + for (int i = 0; i < provisionedHosts.size(); i++) { + tester.makeReadyChildren(1, i + 1, NodeResources.unspecified(), NodeType.config, + provisionedHosts.get(i).hostname(), (nodeIndex) -> "cfg" + nodeIndex); + } + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + + // Expected number of hosts and children are provisioned + NodeList allNodes = tester.nodeRepository().nodes().list(); + NodeList configHosts = allNodes.nodeType(NodeType.confighost); + NodeList configNodes = allNodes.nodeType(NodeType.config); + assertEquals(3, configHosts.size()); + assertEquals(3, configNodes.size()); + String hostnameToRemove = provisionedHosts.get(1).hostname(); + Supplier<Node> hostToRemove = () -> tester.nodeRepository().nodes().node(hostnameToRemove).get(); + Supplier<Node> nodeToRemove = () -> tester.nodeRepository().nodes().node(configNodes.childrenOf(hostnameToRemove).first().get().hostname()).get(); + + // Retire and deprovision host + tester.nodeRepository().nodes().deprovision(hostToRemove.get(), Agent.system, tester.clock().instant()); + tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.system, getClass().getSimpleName()); + assertSame("Host moves to parked", Node.State.parked, hostToRemove.get().state()); + assertSame("Node remains active", Node.State.active, nodeToRemove.get().state()); + assertTrue("Node wants to retire", nodeToRemove.get().status().wantToRetire()); + + // Redeployment of config server application retires node + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + assertTrue("Redeployment retires node", nodeToRemove.get().allocation().get().membership().retired()); + + // Config server becomes removable (done by RetiredExpirer in a real system) and redeployment moves it + // to inactive + tester.nodeRepository().nodes().setRemovable(configSrvApp, List.of(nodeToRemove.get())); + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + assertEquals("Node moves to inactive", Node.State.inactive, nodeToRemove.get().state()); + + // Node is completely removed (done by InactiveExpirer and host-admin in a real system) + Node inactiveConfigServer = nodeToRemove.get(); + int removedIndex = inactiveConfigServer.allocation().get().membership().index(); + tester.nodeRepository().nodes().removeRecursively(inactiveConfigServer, true); + assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.config).size()); + + // Host is removed + dynamicProvisioningTester.maintainer.maintain(); + assertEquals(2, tester.nodeRepository().nodes().list().nodeType(NodeType.confighost).size()); + + // Next deployment starts provisioning a new host and child + try { + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + fail("Expected provisioning to fail"); + } catch (ParentHostUnavailableException ignored) {} + Node newNode = tester.nodeRepository().nodes().list(Node.State.reserved).nodeType(NodeType.config).first().get(); + + // Resume provisioning and activate host + dynamicProvisioningTester.maintainer.maintain(); + List<ProvisionedHost> newHosts = dynamicProvisioningTester.hostProvisioner.provisionedHosts(); + assertEquals(1, newHosts.size()); + tester.nodeRepository().nodes().setReady(newHosts.get(0).hostHostname(), Agent.operator, getClass().getSimpleName()); + tester.prepareAndActivateInfraApplication(hostApp, NodeType.confighost); + assertEquals(3, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); + + // Redeployment of config server app actives new node + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + newNode = tester.nodeRepository().nodes().node(newNode.hostname()).get(); + assertSame(Node.State.active, newNode.state()); + assertEquals("Removed index is reused", removedIndex, newNode.allocation().get().membership().index()); + + // Next redeployment does nothing + NodeList nodesBefore = tester.nodeRepository().nodes().list().nodeType(NodeType.config); + tester.prepareAndActivateInfraApplication(configSrvApp, NodeType.config); + NodeList nodesAfter = tester.nodeRepository().nodes().list().nodeType(NodeType.config); + assertEquals(nodesBefore, nodesAfter); + } + private void assertCfghost3IsActive(DynamicProvisioningTester tester) { assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size()); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); @@ -451,17 +538,17 @@ public class DynamicProvisioningMaintainerTest { private final ProvisioningTester provisioningTester; public DynamicProvisioningTester() { - this(Cloud.builder().dynamicProvisioning(true).build()); + this(Cloud.builder().dynamicProvisioning(true).build(), new MockNameResolver()); } - public DynamicProvisioningTester(Cloud cloud) { - MockNameResolver nameResolver = new MockNameResolver(); - this.hostProvisioner = new MockHostProvisioner(flavors, nameResolver); + public DynamicProvisioningTester(Cloud cloud, MockNameResolver nameResolver) { + this.hostProvisioner = new MockHostProvisioner(flavors.getFlavors(), nameResolver, 0); this.provisioningTester = new ProvisioningTester.Builder().zone(new Zone(cloud, SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName())) .flavors(flavors.getFlavors()) .nameResolver(nameResolver) + .flagSource(flagSource) .hostProvisioner(hostProvisioner) .build(); this.nodeRepository = provisioningTester.nodeRepository(); @@ -529,9 +616,9 @@ public class DynamicProvisioningMaintainerTest { } private long provisionedHostsMatching(NodeResources resources) { - return hostProvisioner.provisionedHosts.stream() - .filter(host -> host.generateHost().resources().compatibleWith(resources)) - .count(); + return hostProvisioner.provisionedHosts().stream() + .filter(host -> host.generateHost().resources().compatibleWith(resources)) + .count(); } private void assertNodesUnchanged() { @@ -542,113 +629,5 @@ public class DynamicProvisioningMaintainerTest { } - static class MockHostProvisioner implements HostProvisioner { - - private final List<ProvisionedHost> provisionedHosts = new ArrayList<>(); - private final NodeFlavors flavors; - private final MockNameResolver nameResolver; - - private int deprovisionedHosts = 0; - private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class); - private Optional<Flavor> provisionHostFlavor = Optional.empty(); - - public MockHostProvisioner(NodeFlavors flavors, MockNameResolver nameResolver) { - this.flavors = flavors; - this.nameResolver = nameResolver; - } - - public MockHostProvisioner provisionSharedHost(String flavorName) { - provisionHostFlavor = Optional.of(flavors.getFlavorOrThrow(flavorName)); - return this; - } - - @Override - public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, - ApplicationId applicationId, Version osVersion, HostSharing sharing) { - Flavor hostFlavor = provisionHostFlavor - .orElseGet(() -> flavors.getFlavors().stream() - .filter(f -> !f.isDocker()) - .filter(f -> f.resources().compatibleWith(resources)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("No host flavor found satisfying " + resources))); - - List<ProvisionedHost> hosts = new ArrayList<>(); - for (int index : provisionIndexes) { - hosts.add(new ProvisionedHost("host" + index, - "hostname" + index, - hostFlavor, - Optional.empty(), - createAddressesForHost(hostFlavor, index), - resources, - osVersion)); - } - provisionedHosts.addAll(hosts); - return hosts; - } - - private List<Address> createAddressesForHost(Flavor flavor, int hostIndex) { - long numAddresses = Math.max(1, Math.round(flavor.resources().bandwidthGbps())); - return IntStream.range(0, (int) numAddresses) - .mapToObj(i -> new Address("nodename" + hostIndex + "_" + i)) - .collect(Collectors.toList()); - } - - @Override - public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { - if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)"); - assertSame(Node.State.provisioned, host.state()); - List<Node> result = new ArrayList<>(); - result.add(withIpAssigned(host)); - for (var child : children) { - assertSame(Node.State.reserved, child.state()); - result.add(withIpAssigned(child)); - } - return result; - } - - @Override - public void deprovision(Node host) { - if (behaviours.contains(Behaviour.failDeprovisioning)) throw new FatalProvisioningException("Failed to deprovision node"); - provisionedHosts.removeIf(provisionedHost -> provisionedHost.hostHostname().equals(host.hostname())); - deprovisionedHosts++; - } - - private MockHostProvisioner with(Behaviour first, Behaviour... rest) { - this.behaviours = EnumSet.of(first, rest); - return this; - } - - private MockHostProvisioner without(Behaviour first, Behaviour... rest) { - Set<Behaviour> behaviours = new HashSet<>(this.behaviours); - behaviours.removeAll(EnumSet.of(first, rest)); - this.behaviours = behaviours.isEmpty() ? EnumSet.noneOf(Behaviour.class) : EnumSet.copyOf(behaviours); - return this; - } - - private Node withIpAssigned(Node node) { - if (node.parentHostname().isPresent()) return node; - int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); - Set<String> addresses = Set.of("::" + hostIndex + ":0"); - Set<String> ipAddressPool = new HashSet<>(); - if (!behaviours.contains(Behaviour.failDnsUpdate)) { - nameResolver.addRecord(node.hostname(), addresses.iterator().next()); - for (int i = 1; i <= 2; i++) { - String ip = "::" + hostIndex + ":" + i; - ipAddressPool.add(ip); - nameResolver.addRecord(node.hostname() + "-" + i, ip); - } - } - - IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool); - return node.with(node.ipConfig().withPrimary(addresses).withPool(pool)); - } - - enum Behaviour { - failProvisioning, - failDeprovisioning, - failDnsUpdate, - } - - } - } + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index eda744e9ee1..d5699f0cffe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; @@ -18,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import com.yahoo.vespa.orchestrator.OrchestrationException; @@ -28,9 +30,12 @@ import java.time.Duration; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; @@ -64,7 +69,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out tester.advanceTime(Duration.ofMinutes(14)); - new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().nodes().list(Node.State.inactive).size()); NodeList dirty = tester.nodeRepository().nodes().list(Node.State.dirty); assertEquals(2, dirty.size()); @@ -105,7 +110,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out and node is moved to dirty tester.advanceTime(Duration.ofMinutes(14)); - new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run(); NodeList dirty = tester.nodeRepository().nodes().list(Node.State.dirty); assertEquals(2, dirty.size()); @@ -156,7 +161,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out and one node is moved to parked tester.advanceTime(Duration.ofMinutes(11)); // Trigger InactiveExpirer - new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run(); assertEquals(1, tester.nodeRepository().nodes().list(Node.State.parked).size()); } @@ -178,7 +183,7 @@ public class InactiveAndFailedExpirerTest { assertEquals(1, inactiveNodes.size()); // See that nodes are moved to dirty immediately. - new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().nodes().list(Node.State.inactive).size()); NodeList dirty = tester.nodeRepository().nodes().list(Node.State.dirty); assertEquals(1, dirty.size()); @@ -202,8 +207,31 @@ public class InactiveAndFailedExpirerTest { // Nodes marked for deprovisioning are moved to parked tester.patchNodes(inactiveNodes, (node) -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())); tester.advanceTime(Duration.ofMinutes(11)); - new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), Map.of(), new TestMetric()).run(); assertEquals(2, tester.nodeRepository().nodes().list(Node.State.parked).size()); } + @Test + public void inactive_config_server_expires_according_to_custom_timeout() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + InactiveExpirer expirer = new InactiveExpirer(tester.nodeRepository(), Duration.ofHours(1), + Map.of(NodeType.config, Duration.ofMinutes(5)), + new TestMetric()); + NodeList nodes = tester.makeConfigServers(3, "default", Vtag.currentVersion); + Supplier<Node> firstNode = () -> tester.nodeRepository().nodes().node(nodes.first().get().hostname()).get(); + ApplicationId application = firstNode.get().allocation().get().owner(); + + // Retired config server is moved to inactive + tester.nodeRepository().nodes().retire(NodeListFilter.from(firstNode.get()), Agent.system, tester.clock().instant()); + tester.prepareAndActivateInfraApplication(application, NodeType.config); + assertSame(Node.State.inactive, firstNode.get().state()); + expirer.maintain(); + assertSame(Node.State.inactive, firstNode.get().state()); + + // Config server expires + tester.clock().advance(Duration.ofMinutes(5).plus(Duration.ofSeconds(1))); + expirer.maintain(); + assertSame(Node.State.dirty, firstNode.get().state()); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java index 131c02015a1..4db1b86419b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java @@ -14,22 +14,20 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; -import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing; +import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import org.junit.Test; import java.time.Instant; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -73,7 +71,7 @@ public class DynamicDockerProvisionTest { mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); - verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), resources, application1, + verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), NodeType.host, resources, application1, Version.emptyVersion, HostSharing.any); // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes @@ -99,7 +97,7 @@ public class DynamicDockerProvisionTest { ApplicationId application3 = ProvisioningTester.applicationId(); mockHostProvisioner(hostProvisioner, "large", 3, application3); prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, resources); - verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), resources, application3, + verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), NodeType.host, resources, application3, Version.emptyVersion, HostSharing.exclusive); // Total of 20 nodes should now be in node-repo, 8 active hosts and 12 active nodes @@ -429,7 +427,7 @@ public class DynamicDockerProvisionTest { doAnswer(invocation -> { Flavor hostFlavor = tester.nodeRepository().flavors().getFlavorOrThrow(hostFlavorName); List<Integer> provisionIndexes = (List<Integer>) invocation.getArguments()[0]; - NodeResources nodeResources = (NodeResources) invocation.getArguments()[1]; + NodeResources nodeResources = (NodeResources) invocation.getArguments()[2]; return provisionIndexes.stream() .map(hostIndex -> { @@ -451,52 +449,7 @@ public class DynamicDockerProvisionTest { return provisionedHost; }) .collect(Collectors.toList()); - }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any()); - } - - private static class MockHostProvisioner implements HostProvisioner { - - private final List<Flavor> hostFlavors; - private final int memoryTaxGb; - - public MockHostProvisioner(List<Flavor> hostFlavors, int memoryTaxGb) { - this.hostFlavors = List.copyOf(hostFlavors); - this.memoryTaxGb = memoryTaxGb; - } - - @Override - public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, - ApplicationId applicationId, Version osVersion, HostSharing sharing) { - Optional<Flavor> hostFlavor = hostFlavors.stream().filter(f -> compatible(f, resources)).findFirst(); - if (hostFlavor.isEmpty()) - throw new OutOfCapacityException("No host flavor matches " + resources); - return provisionIndexes.stream() - .map(i -> new ProvisionedHost("id-" + i, "host-" + i, hostFlavor.get(), Optional.empty(), - List.of(new Address("host-" + i + "-1")), resources, osVersion)) - .collect(Collectors.toList()); - } - - private boolean compatible(Flavor hostFlavor, NodeResources resources) { - NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); - - if (hostFlavor.resources().storageType() == NodeResources.StorageType.remote - && hostFlavor.resources().diskGb() >= resources.diskGb()) - resourcesToVerify = resourcesToVerify.withDiskGb(hostFlavor.resources().diskGb()); - if (hostFlavor.resources().bandwidthGbps() >= resources.bandwidthGbps()) - resourcesToVerify = resourcesToVerify.withBandwidthGbps(hostFlavor.resources().bandwidthGbps()); - return hostFlavor.resources().compatibleWith(resourcesToVerify); - } - - @Override - public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { - throw new RuntimeException("Not implemented: provision"); - } - - @Override - public void deprovision(Node host) { - throw new RuntimeException("Not implemented: deprovision"); - } - + }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any(), any()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 0986f2954a7..c269b4642ea 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -33,7 +33,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -1017,7 +1016,7 @@ public class ProvisioningTest { private Set<HostSpec> prepare(ApplicationId application, ProvisioningTester tester, ClusterSpec cluster, int nodeCount, int groups, boolean required, NodeResources nodeResources) { - if (nodeCount == 0) return Collections.emptySet(); // this is a shady practice + if (nodeCount == 0) return Set.of(); // this is a shady practice return new HashSet<>(tester.prepare(application, cluster, nodeCount, groups, required, nodeResources)); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 97baddf93fa..eefbd03ce4e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -227,7 +227,10 @@ public class ProvisioningTester { } public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())).vespaVersion(version).build(); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())) + .vespaVersion(version) + .stateful(nodeType == NodeType.config || nodeType == NodeType.controller) + .build(); Capacity capacity = Capacity.fromRequiredNodeType(nodeType); List<HostSpec> hostSpecs = prepare(application, cluster, capacity); activate(application, hostSpecs); @@ -510,17 +513,18 @@ public class ProvisioningTester { index -> UUID.randomUUID().toString()); } - /** Creates a set of virtual nodes on a single parent host */ - public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, String parentHostname, - Function<Integer, String> nodeNamer) { + /** Create one or more child nodes on given parent host */ + public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, NodeType nodeType, + String parentHostname, Function<Integer, String> nodeNamer) { + if (nodeType.isHost()) throw new IllegalArgumentException("Non-child node type: " + nodeType); List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { String hostname = nodeNamer.apply(i); IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), Set.of()); - - Node.Builder builder = Node.create("node-id", ipConfig, hostname, new Flavor(resources), NodeType.tenant); - builder.parentHostname(parentHostname); - nodes.add(builder.build()); + Node node = Node.create("node-id", ipConfig, hostname, new Flavor(resources), nodeType) + .parentHostname(parentHostname) + .build(); + nodes.add(node); } nodes = nodeRepository.nodes().addNodes(nodes, Agent.system); nodes = nodeRepository.nodes().deallocate(nodes, Agent.system, getClass().getSimpleName()); @@ -528,6 +532,12 @@ public class ProvisioningTester { return nodes; } + /** Create one or more child nodes on given parent host */ + public List<Node> makeReadyChildren(int count, int startIndex, NodeResources resources, String parentHostname, + Function<Integer, String> nodeNamer) { + return makeReadyChildren(count, startIndex, resources, NodeType.tenant, parentHostname, nodeNamer); + } + public void activateTenantHosts() { prepareAndActivateInfraApplication(applicationId(), NodeType.host); } |