diff options
author | Harald Musum <musum@yahooinc.com> | 2023-11-03 11:50:03 +0100 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-11-03 11:50:03 +0100 |
commit | be72f2e85fd0770abaea981c8488bf22271a6f2a (patch) | |
tree | cfb4fc5eface95341428a071a45d6b737baa4fd2 /node-repository | |
parent | 96f6abe9caa338074ee39cb2fd566d3efff464c9 (diff) |
Add test of reducing container node count
Diffstat (limited to 'node-repository')
-rw-r--r-- | node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java | 151 |
1 files changed, 92 insertions, 59 deletions
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 5927cb43c3a..abcef421b4c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -40,9 +40,14 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.yahoo.config.provision.ClusterSpec.Type.container; +import static com.yahoo.config.provision.ClusterSpec.Type.content; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static com.yahoo.config.provision.NodeResources.StorageType.remote; +import static com.yahoo.vespa.hosted.provision.Node.State.active; +import static com.yahoo.vespa.hosted.provision.Node.State.dirty; +import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester.applicationId; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -60,35 +65,35 @@ public class DynamicProvisioningTest { var tester = tester(true); assertEquals(0, tester.nodeRepository().nodes().list().size()); - ApplicationId application1 = ProvisioningTester.applicationId("application1"); + ApplicationId application1 = applicationId("application1"); NodeResources resources = new NodeResources(1, 4, 10, 1); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources, tester); // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes assertEquals(8, tester.nodeRepository().nodes().list().size()); - assertEquals(4, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host).size()); + assertEquals(4, tester.nodeRepository().nodes().list(active).nodeType(NodeType.host).size()); assertEquals(Set.of("host100-1", "host101-1", "host102-1", "host103-1"), - tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).hostnames()); + tester.nodeRepository().nodes().list(active).nodeType(NodeType.tenant).hostnames()); // Deploy new application - ApplicationId application2 = ProvisioningTester.applicationId("application2"); + ApplicationId application2 = applicationId("application2"); prepareAndActivate(application2, clusterSpec("mycluster"), 4, 1, resources, tester); // Total of 12 nodes should now be in node-repo, 4 active hosts and 8 active nodes assertEquals(12, tester.nodeRepository().nodes().list().size()); - assertEquals(4, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host).size()); + assertEquals(4, tester.nodeRepository().nodes().list(active).nodeType(NodeType.host).size()); assertEquals(Set.of("host100-1", "host100-2", "host101-1", "host101-2", "host102-1", "host102-2", "host103-1", "host103-2"), - tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).hostnames()); + tester.nodeRepository().nodes().list(active).nodeType(NodeType.tenant).hostnames()); // Deploy new exclusive application - ApplicationId application3 = ProvisioningTester.applicationId("application3"); + ApplicationId application3 = applicationId("application3"); NodeResources exclusiveResources = new NodeResources(2, 10, 20, 1); prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, exclusiveResources, tester); // Total of 20 nodes should now be in node-repo, 8 active hosts and 12 active nodes assertEquals(20, tester.nodeRepository().nodes().list().size()); - assertEquals(8, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host).size()); - assertEquals(12, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); + assertEquals(8, tester.nodeRepository().nodes().list(active).nodeType(NodeType.host).size()); + assertEquals(12, tester.nodeRepository().nodes().list(active).nodeType(NodeType.tenant).size()); } @Test @@ -98,15 +103,15 @@ public class DynamicProvisioningTest { NodeResources initialResources = new NodeResources(4, 80, 100, 1); NodeResources smallResources = new NodeResources(2, 20, 50, 1); - ApplicationId application1 = ProvisioningTester.applicationId(); + ApplicationId application1 = applicationId(); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, initialResources, tester); - ApplicationId application2 = ProvisioningTester.applicationId(); + ApplicationId application2 = applicationId(); prepareAndActivate(application2, clusterSpec("mycluster", true), 4, 1, initialResources, tester); // Total of 16 nodes should now be in node-repo, 8 active hosts and 8 active nodes assertEquals(16, tester.nodeRepository().nodes().list().size()); - assertEquals(8, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); + assertEquals(8, tester.nodeRepository().nodes().list(active).nodeType(NodeType.tenant).size()); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, smallResources, tester); prepareAndActivate(application2, clusterSpec("mycluster", true), 4, 1, smallResources, tester); @@ -114,8 +119,8 @@ public class DynamicProvisioningTest { // 24 nodes: 4 shared hosts with 4 app1 nodes + 8 exclusive hosts with 8 nodes of app2, 4 of which are retired NodeList nodes = tester.nodeRepository().nodes().list(); assertEquals(24, nodes.size()); - assertEquals(12, nodes.nodeType(NodeType.host).state(Node.State.active).size()); - assertEquals(12, nodes.nodeType(NodeType.tenant).state(Node.State.active).size()); + assertEquals(12, nodes.nodeType(NodeType.host).state(active).size()); + assertEquals(12, nodes.nodeType(NodeType.tenant).state(active).size()); assertEquals(4, nodes.retired().size()); } @@ -126,17 +131,17 @@ public class DynamicProvisioningTest { NodeResources highResources = new NodeResources(4, 80, 100, 1); NodeResources lowResources = new NodeResources(2, 20, 50, 1); - ApplicationId application = ProvisioningTester.applicationId(); + ApplicationId application = applicationId(); prepareAndActivate(application, clusterSpec("mycluster", true), 2, 1, highResources, tester); // Total of 4 nodes should now be in node-repo, 2 active hosts and 2 active nodes. assertEquals(4, tester.nodeRepository().nodes().list().size()); - assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); + assertEquals(2, tester.nodeRepository().nodes().list(active).nodeType(NodeType.tenant).size()); // Redeploying the application causes no changes at all. prepareAndActivate(application, clusterSpec("mycluster", true), 2, 1, highResources, tester); assertEquals(4, tester.nodeRepository().nodes().list().size()); - assertEquals(2, tester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); + assertEquals(2, tester.nodeRepository().nodes().list(active).nodeType(NodeType.tenant).size()); // Deploying with a smaller node flavour causes new, smaller hosts to be provisioned. prepareAndActivate(application, clusterSpec("mycluster", true), 2, 1, lowResources, tester); @@ -144,8 +149,8 @@ public class DynamicProvisioningTest { // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes, of which 2 are retired. NodeList nodes = tester.nodeRepository().nodes().list(); assertEquals(8, nodes.size()); - assertEquals(4, nodes.nodeType(NodeType.host).state(Node.State.active).size()); - assertEquals(4, nodes.nodeType(NodeType.tenant).state(Node.State.active).size()); + assertEquals(4, nodes.nodeType(NodeType.host).state(active).size()); + assertEquals(4, nodes.nodeType(NodeType.tenant).state(active).size()); assertEquals(2, nodes.retired().size()); // Remove the child nodes, and redeploy with the original flavour. This should reuse the existing hosts. @@ -155,8 +160,8 @@ public class DynamicProvisioningTest { // Total of 6 nodes should now be in node-repo, 4 active hosts and 2 active nodes. nodes = tester.nodeRepository().nodes().list(); assertEquals(6, nodes.size()); - assertEquals(4, nodes.nodeType(NodeType.host).state(Node.State.active).size()); - assertEquals(2, nodes.nodeType(NodeType.tenant).state(Node.State.active).size()); + assertEquals(4, nodes.nodeType(NodeType.host).state(active).size()); + assertEquals(2, nodes.nodeType(NodeType.tenant).state(active).size()); assertEquals(0, nodes.retired().size()); // Deploy again with high resources. @@ -164,8 +169,8 @@ public class DynamicProvisioningTest { // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes. nodes = tester.nodeRepository().nodes().list(); assertEquals(8, nodes.size()); - assertEquals(4, nodes.nodeType(NodeType.host).state(Node.State.active).size()); - assertEquals(4, nodes.nodeType(NodeType.tenant).state(Node.State.active).size()); + assertEquals(4, nodes.nodeType(NodeType.host).state(active).size()); + assertEquals(4, nodes.nodeType(NodeType.tenant).state(active).size()); } @Test @@ -176,27 +181,27 @@ public class DynamicProvisioningTest { NodeResources resources = new NodeResources(2, 4, 10, 4); - ApplicationId application1 = ProvisioningTester.applicationId(); + ApplicationId application1 = applicationId(); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources, tester); - ApplicationId application2 = ProvisioningTester.applicationId(); + ApplicationId application2 = applicationId(); prepareAndActivate(application2, clusterSpec("mycluster"), 3, 1, resources, tester); - ApplicationId application3 = ProvisioningTester.applicationId(); + ApplicationId application3 = applicationId(); prepareAndActivate(application3, clusterSpec("mycluster"), 3, 1, resources, tester); assertEquals(4, tester.nodeRepository().nodes().list().nodeType(NodeType.tenant).stream().map(Node::parentHostname).distinct().count()); - ApplicationId application4 = ProvisioningTester.applicationId(); + ApplicationId application4 = applicationId(); prepareAndActivate(application4, clusterSpec("mycluster"), 3, 1, resources, tester); assertEquals(5, tester.nodeRepository().nodes().list().nodeType(NodeType.tenant).stream().map(Node::parentHostname).distinct().count()); } @Test public void does_not_allocate_container_nodes_to_shared_hosts() { - assertHostSharing(Environment.prod, ClusterSpec.Type.container, false); - assertHostSharing(Environment.prod, ClusterSpec.Type.content, true); - assertHostSharing(Environment.staging, ClusterSpec.Type.container, true); - assertHostSharing(Environment.staging, ClusterSpec.Type.content, true); + assertHostSharing(Environment.prod, container, false); + assertHostSharing(Environment.prod, content, true); + assertHostSharing(Environment.staging, container, true); + assertHostSharing(Environment.staging, content, true); } private void assertHostSharing(Environment environment, ClusterSpec.Type clusterType, boolean expectShared) { @@ -206,7 +211,7 @@ public class DynamicProvisioningTest { tester.makeReadyHosts(2, new NodeResources(12, 12, 200, 12)); tester.flagSource().withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(new HostResources(4.0, 16.0, 50.0, 0.3, "fast", "local", null, 10, "x86_64"))), SharedHost.class); - ApplicationId application = ProvisioningTester.applicationId(); + ApplicationId application = applicationId(); ClusterSpec cluster = ClusterSpec.request(clusterType, ClusterSpec.Id.from("default")).vespaVersion("6.42").build(); tester.prepare(application, cluster, 2, 1, new NodeResources(2., 10., 20, 1)); assertEquals(expectShared ? 2 : 4, tester.nodeRepository().nodes().list().nodeType(NodeType.host).size()); @@ -216,7 +221,7 @@ public class DynamicProvisioningTest { public void retires_on_exclusivity_violation() { var tester = tester(false); tester.flagSource().withJacksonFlag(PermanentFlags.SHARED_HOST.id(), new SharedHost(List.of(new HostResources(1., 1., 1., 1., "fast", "local", null, 10, "x86_64"))), SharedHost.class); - ApplicationId application1 = ProvisioningTester.applicationId(); + ApplicationId application1 = applicationId(); NodeResources resources = new NodeResources(4, 80, 100, 1); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources, tester); NodeList initialNodes = tester.nodeRepository().nodes().list().owner(application1); @@ -239,7 +244,7 @@ public class DynamicProvisioningTest { NodeList exclusiveViolators = nodes.owner(application1).not().retired().first(2); List<Node> parents = exclusiveViolators.mapToList(node -> nodes.parentOf(node).get()); tester.patchNode(parents.get(0), node -> node.withProvisionedForApplicationId(ApplicationId.defaultId())); - tester.patchNode(parents.get(1), node -> node.withExclusiveToClusterType(ClusterSpec.Type.container)); + tester.patchNode(parents.get(1), node -> node.withExclusiveToClusterType(container)); prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, smallerExclusiveResources, tester); assertEquals(10, tester.nodeRepository().nodes().list().owner(application1).size()); @@ -250,7 +255,7 @@ public class DynamicProvisioningTest { public void node_indices_are_unique_even_when_a_node_is_left_in_reserved_state() { var tester = tester(true); NodeResources resources = new NodeResources(10, 10, 10, 10); - ApplicationId app = ProvisioningTester.applicationId(); + ApplicationId app = applicationId(); Function<Node, Node> retireNode = node -> tester.patchNode(node, (n) -> n.withWantToRetire(true, Agent.system, Instant.now())); Function<Integer, Node> getNodeInGroup = group -> tester.nodeRepository().nodes().list().owner(app).stream() @@ -295,8 +300,8 @@ public class DynamicProvisioningTest { tester.activateTenantHosts(); - ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ApplicationId app1 = applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); // Deploy using real memory amount (17) try { @@ -333,32 +338,60 @@ public class DynamicProvisioningTest { .flagSource(flagSource) .build(); - ApplicationId app = ProvisioningTester.applicationId("a1"); - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); + ApplicationId app = applicationId("a1"); + ClusterSpec cluster = ClusterSpec.request(content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 8, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any))); - hostProvisioner.setHostFlavor("x86", ClusterSpec.Type.content); + hostProvisioner.setHostFlavor("x86", content); tester.activate(app, cluster, capacity); NodeList nodes = tester.nodeRepository().nodes().list(); - assertEquals(4, nodes.owner(app).state(Node.State.active).size()); - assertEquals(Set.of("x86"), nodes.parentsOf(nodes.owner(app).state(Node.State.active)).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); + assertEquals(4, nodes.owner(app).state(active).size()); + assertEquals(Set.of("x86"), nodes.parentsOf(nodes.owner(app).state(active)).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); - hostProvisioner.setHostFlavor("arm", ClusterSpec.Type.content); + hostProvisioner.setHostFlavor("arm", content); flagSource.withStringFlag(PermanentFlags.HOST_FLAVOR.id(), "arm"); tester.activate(app, cluster, capacity); nodes = tester.nodeRepository().nodes().list(); - assertEquals(4, nodes.owner(app).state(Node.State.active).retired().size()); - assertEquals(4, nodes.owner(app).state(Node.State.active).not().retired().size()); - assertEquals(Set.of("x86"), nodes.parentsOf(tester.getNodes(app, Node.State.active).retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); - assertEquals(Set.of("arm"), nodes.parentsOf(tester.getNodes(app, Node.State.active).not().retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); + assertEquals(4, nodes.owner(app).state(active).retired().size()); + assertEquals(4, nodes.owner(app).state(active).not().retired().size()); + assertEquals(Set.of("x86"), nodes.parentsOf(tester.getNodes(app, active).retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); + assertEquals(Set.of("arm"), nodes.parentsOf(tester.getNodes(app, active).not().retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); flagSource.removeFlag(PermanentFlags.HOST_FLAVOR.id()); // Resetting flag does not move the nodes back tester.activate(app, cluster, capacity); nodes = tester.nodeRepository().nodes().list(); - assertEquals(4, nodes.owner(app).state(Node.State.active).retired().size()); - assertEquals(4, nodes.owner(app).state(Node.State.active).not().retired().size()); - assertEquals(Set.of("x86"), nodes.parentsOf(tester.getNodes(app, Node.State.active).retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); - assertEquals(Set.of("arm"), nodes.parentsOf(tester.getNodes(app, Node.State.active).not().retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); + assertEquals(4, nodes.owner(app).state(active).retired().size()); + assertEquals(4, nodes.owner(app).state(active).not().retired().size()); + assertEquals(Set.of("x86"), nodes.parentsOf(tester.getNodes(app, active).retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); + assertEquals(Set.of("arm"), nodes.parentsOf(tester.getNodes(app, active).not().retired()).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); + } + + + @Test + public void reduces_container_node_count() { + List<Flavor> flavors = List.of(new Flavor("default", new NodeResources(2, 8, 50, 0.1, fast, local, Architecture.x86_64))); + MockHostProvisioner hostProvisioner = new MockHostProvisioner(flavors); + ProvisioningTester tester = new ProvisioningTester.Builder() + .dynamicProvisioning(true, false) + .flavors(flavors) + .hostProvisioner(hostProvisioner) + .build(); + + ApplicationId app = applicationId("a1"); + ClusterSpec cluster = ClusterSpec.request(container, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); + Capacity capacity = Capacity.from(resources(4, 1, 2, 8, 50)); + + tester.activate(app, cluster, capacity); + NodeList nodes = tester.nodeRepository().nodes().list(); + assertEquals(4, nodes.owner(app).state(active).size()); + assertEquals(0, nodes.owner(app).state(dirty).size()); + + // Go from 4 to 2 nodes, 2 nodes will go directly to dirty + capacity = Capacity.from(resources(2, 1, 2, 8, 50)); + tester.activate(app, cluster, capacity); + nodes = tester.nodeRepository().nodes().list(); + assertEquals(2, nodes.owner(app).state(active).size()); + assertEquals(2, nodes.owner(app).state(dirty).size()); } @Test @@ -377,8 +410,8 @@ public class DynamicProvisioningTest { tester.activateTenantHosts(); - ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ApplicationId app1 = applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); // Limits where each number is within flavor limits but which don't contain any flavor leads to an error try { @@ -452,8 +485,8 @@ public class DynamicProvisioningTest { tester.activateTenantHosts(); - ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ApplicationId app1 = applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, local), resources(6, 3, 3, 25, 400, fast, local))); @@ -487,8 +520,8 @@ public class DynamicProvisioningTest { tester.activateTenantHosts(); - ApplicationId app1 = ProvisioningTester.applicationId("app1"); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + ApplicationId app1 = applicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(container, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 200, fast, StorageType.any), resources(6, 3, 3, 25, 400, fast, StorageType.any))); @@ -515,7 +548,7 @@ public class DynamicProvisioningTest { NodeResources resources = new NodeResources(4, 16, 125, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, NodeResources.Architecture.x86_64, new NodeResources.GpuResources(1, 16)); - tester.prepare(ProvisioningTester.applicationId(), ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")) + tester.prepare(applicationId(), ClusterSpec.request(container, ClusterSpec.Id.from("id1")) .vespaVersion("8.0").build(), 2, 1, resources); } @@ -555,7 +588,7 @@ public class DynamicProvisioningTest { } private static ClusterSpec clusterSpec(String clusterId, boolean exclusive) { - return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").exclusive(exclusive).build(); + return ClusterSpec.request(content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").exclusive(exclusive).build(); } private static ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { |