// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.maintenance.ReservationExpirer; import com.yahoo.vespa.hosted.provision.maintenance.TestMetric; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.duper.ConfigServerHostApplication; import com.yahoo.vespa.service.duper.InfraApplication; import org.junit.Test; import java.time.Duration; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** * Various allocation sequence scenarios * * @author bratseth * @author mpolden */ public class ProvisioningTest { private static final NodeResources defaultResources = new NodeResources(1, 4, 10, 4); @Test public void application_deployment_constant_application_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application1 = ProvisioningTester.applicationId(); ApplicationId application2 = ProvisioningTester.applicationId(); tester.makeReadyHosts(21, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); tester.activate(application1, state1.allHosts); // redeploy SystemState state2 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); state2.assertEquals(state1); tester.activate(application1, state2.allHosts); // deploy another application SystemState state1App2 = prepare(application2, 2, 2, 3, 3, defaultResources, tester); assertFalse("Hosts to different apps are disjunct", state1App2.allHosts.removeAll(state1.allHosts)); tester.activate(application2, state1App2.allHosts); // prepare twice SystemState state3 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); SystemState state4 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); state3.assertEquals(state2); state4.assertEquals(state3); tester.activate(application1, state4.allHosts); // remove nodes before deploying SystemState state5 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); HostSpec removed = tester.removeOne(state5.allHosts); tester.activate(application1, state5.allHosts); assertEquals(removed.hostname(), tester.nodeRepository().nodes().list(Node.State.inactive).owner(application1).first().get().hostname()); // remove some of the clusters SystemState state6 = prepare(application1, 0, 2, 0, 3, defaultResources, tester); tester.activate(application1, state6.allHosts); assertEquals(5, tester.getNodes(application1, Node.State.active).size()); assertEquals(3, tester.getNodes(application1, Node.State.inactive).size()); // delete app NodeList previousNodes = tester.getNodes(application1); tester.remove(application1); assertEquals(previousNodes.hostnames(), tester.nodeRepository().nodes().list(Node.State.dirty).owner(application1).hostnames()); assertEquals(0, tester.getNodes(application1, Node.State.active).size()); assertTrue(tester.nodeRepository().applications().get(application1).isEmpty()); // other application is unaffected assertEquals(state1App2.hostNames(), tester.nodeRepository().nodes().list(Node.State.active).owner(application2).hostnames()); // fail a node from app2 and make sure it does not get inactive nodes from first HostSpec failed = tester.removeOne(state1App2.allHosts); tester.fail(failed); assertEquals(9, tester.getNodes(application2, Node.State.active).size()); SystemState state2App2 = prepare(application2, 2, 2, 3, 3, defaultResources, tester); assertFalse("Hosts to different apps are disjunct", state2App2.allHosts.removeAll(state1.allHosts)); assertEquals("A new node was reserved to replace the failed one", 10, state2App2.allHosts.size()); assertFalse("The new host is not the failed one", state2App2.allHosts.contains(failed)); tester.activate(application2, state2App2.allHosts); // deploy first app again tester.nodeRepository().nodes().setReady(tester.nodeRepository().nodes().list(Node.State.dirty).asList(), Agent.system, "recycled"); SystemState state7 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); state7.assertEquals(state1); tester.activate(application1, state7.allHosts); assertEquals(0, tester.getNodes(application1, Node.State.inactive).size()); // restart HostFilter allFilter = HostFilter.all(); HostFilter hostFilter = HostFilter.hostname(state6.allHosts.iterator().next().hostname()); HostFilter clusterTypeFilter = HostFilter.clusterType(ClusterSpec.Type.container); HostFilter clusterIdFilter = HostFilter.clusterId(ClusterSpec.Id.from("container1")); tester.provisioner().restart(application1, allFilter); tester.provisioner().restart(application1, hostFilter); tester.provisioner().restart(application1, clusterTypeFilter); tester.provisioner().restart(application1, clusterIdFilter); tester.assertRestartCount(application1, allFilter, hostFilter, clusterTypeFilter, clusterIdFilter); } @Test public void nodeVersionIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); // deploy ApplicationId application1 = ProvisioningTester.applicationId(); SystemState state1 = prepare(application1, 1, 1, 1, 1, defaultResources, tester); tester.activate(application1, state1.allHosts); HostSpec host1 = state1.container0.iterator().next(); assertFalse(host1.version().isPresent()); Node node1 = tester.nodeRepository().nodes().node(host1.hostname()).get(); tester.nodeRepository().nodes().write(node1.with(node1.status().withVespaVersion(Version.fromString("1.2.3"))), () -> {}); // redeploy SystemState state2 = prepare(application1, 1, 1, 1, 1, defaultResources, tester); tester.activate(application1, state2.allHosts); host1 = state2.container0.iterator().next(); assertEquals(Version.fromString("1.2.3"), host1.version().get()); } @Test public void dockerImageRepoIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); // deploy ApplicationId application1 = ProvisioningTester.applicationId(); SystemState state1 = prepare(application1, tester, 1, 1, 1, 1, defaultResources, "1.2.3"); String dockerImageRepo = "docker.domain.tld/my/image"; prepare(application1, tester, 1, 1, 1 , 1 , false, defaultResources, "1.2.3"); tester.activate(application1, state1.allHosts); HostSpec host1 = state1.container0.iterator().next(); Node node1 = tester.nodeRepository().nodes().node(host1.hostname()).get(); DockerImage dockerImage = DockerImage.fromString(dockerImageRepo).withTag(Version.fromString("1.2.3")); tester.nodeRepository().nodes().write(node1.with(node1.status().withContainerImage(dockerImage)), () -> {}); // redeploy SystemState state2 = prepare(application1, tester, 1, 1, 1 ,1 , false, defaultResources, "1.2.3"); tester.activate(application1, state2.allHosts); host1 = state2.container0.iterator().next(); node1 = tester.nodeRepository().nodes().node(host1.hostname()).get(); assertEquals(dockerImage, node1.status().containerImage().get()); } @Test public void application_deployment_variable_application_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application1 = ProvisioningTester.applicationId(); tester.makeReadyHosts(30, defaultResources); tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); tester.activate(application1, state1.allHosts); // redeploy with increased sizes SystemState state2 = prepare(application1, 3, 4, 4, 5, defaultResources, tester); state2.assertExtends(state1); assertEquals("New nodes are reserved", 6, tester.getNodes(application1, Node.State.reserved).size()); tester.activate(application1, state2.allHosts); // decrease again SystemState state3 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); tester.activate(application1, state3.allHosts); assertEquals("Superfluous container nodes are dirtyed", 3-2 + 4-2, tester.nodeRepository().nodes().list(Node.State.dirty).size()); assertEquals("Superfluous content nodes are retired", 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size()); // increase even more, and remove one node before deploying SystemState state4 = prepare(application1, 4, 5, 5, 6, defaultResources, tester); assertEquals("Inactive nodes are reused", 0, tester.getNodes(application1, Node.State.inactive).size()); assertEquals("Earlier retired nodes are not unretired before activate", 4-3 + 5-3, tester.getNodes(application1, Node.State.active).retired().size()); assertEquals("New and inactive nodes are reserved", 4 + 3, tester.getNodes(application1, Node.State.reserved).size()); // Remove a retired host from one of the content clusters (which one is random depending on host names) HostSpec removed = state4.removeHost(tester.getNodes(application1, Node.State.active).retired().asList().get(0).hostname()); tester.activate(application1, state4.allHosts); assertEquals("Retired active removed when activating became inactive", 1, tester.getNodes(application1, Node.State.inactive).asList().size()); assertEquals(removed.hostname(), tester.getNodes(application1, Node.State.inactive).asList().get(0).hostname()); assertEquals("Earlier retired nodes are unretired on activate", 0, tester.getNodes(application1, Node.State.active).retired().size()); // decrease again SystemState state5 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); tester.activate(application1, state5.allHosts); assertEquals("Superfluous container nodes are also dirtyed", 4-2 + 5-2 + 1 + 4-2, tester.nodeRepository().nodes().list(Node.State.dirty).size()); assertEquals("Superfluous content nodes are retired", 5-3 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); // increase content slightly SystemState state6 = prepare(application1, 2, 2, 4, 3, defaultResources, tester); tester.activate(application1, state6.allHosts); assertEquals("One content node is unretired", 5-4 + 6-3 - 1, tester.getNodes(application1, Node.State.active).retired().size()); // Then reserve more SystemState state7 = prepare(application1, 8, 2, 2, 2, defaultResources, tester); // delete app tester.remove(application1); assertEquals(0, tester.getNodes(application1, Node.State.active).size()); assertEquals(0, tester.getNodes(application1, Node.State.reserved).size()); } @Test public void application_deployment_multiple_flavors() { NodeResources small = new NodeResources(1, 4, 10, 0.3); NodeResources large = new NodeResources(8, 8, 40, 0.3); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application1 = ProvisioningTester.applicationId(); tester.makeReadyHosts(12, small); tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); tester.activate(application1, state1.allHosts); // redeploy with reduced size (to cause us to have retired nodes before switching resources) SystemState state2 = prepare(application1, 2, 2, 3, 3, small, tester); tester.activate(application1, state2.allHosts); tester.makeReadyHosts(16, large); tester.activateTenantHosts(); // redeploy with increased sizes and new flavor SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester); assertEquals("New nodes are reserved", 16, tester.nodeRepository().nodes().list(Node.State.reserved).owner(application1).size()); tester.activate(application1, state3.allHosts); assertEquals("small container nodes are retired because we are swapping the entire cluster", 2 + 2, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.container).resources(small).size()); assertEquals("'small content nodes are retired", 4 + 4, tester.getNodes(application1, Node.State.active).retired().type(ClusterSpec.Type.content).resources(small).size()); assertEquals("No large content nodes are retired", 0, tester.getNodes(application1, Node.State.active).retired().resources(large).size()); } @Test public void host_flavors_at_least_twice_as_large_as_preferred() { NodeResources small = new NodeResources(1, 4, 10, 0.3); NodeResources large = new NodeResources(8, 8, 40, 0.3); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application1 = ProvisioningTester.applicationId(); tester.makeReadyHosts(12, small); tester.makeReadyHosts(12, large); tester.activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); tester.activate(application1, state1.allHosts); tester.nodeRepository().nodes().list().owner(application1) .forEach(n -> assertEquals(large, tester.nodeRepository().nodes().node(n.parentHostname().get()).get().resources())); } @Test public void application_deployment_above_then_at_capacity_limit() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application1 = ProvisioningTester.applicationId(); tester.makeReadyHosts(5, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); tester.activate(application1, state1.allHosts); // redeploy a too large application try { SystemState state2 = prepare(application1, 3, 0, 3, 0, defaultResources, tester); fail("Expected out of capacity exception"); } catch (OutOfCapacityException expected) { } // deploy first state again SystemState state3 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); tester.activate(application1, state3.allHosts); } @Test public void dev_deployment_node_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); ApplicationId application = ProvisioningTester.applicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void requested_resources_info_is_retained() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(13, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); ApplicationId application = ProvisioningTester.applicationId(); { // Deploy with disk-speed any and make sure that information is retained SystemState state = prepare(application, 0, 0, 3, 3, defaultResources.justNumbers(), tester); assertEquals(6, state.allHosts.size()); tester.activate(application, state.allHosts); assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.any)); assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any)); } { // Deploy (with some additional nodes) with disk-speed fast and make sure *that* information is retained // even though it does not lead to new nodes SystemState state = prepare(application, 0, 0, 5, 3, defaultResources, tester); assertEquals(8, state.allHosts.size()); tester.activate(application, state.allHosts); assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.fast)); assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.fast)); } { // Go back to any SystemState state = prepare(application, 0, 0, 5, 3, defaultResources.justNumbers(), tester); assertEquals(8, state.allHosts.size()); tester.activate(application, state.allHosts); assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.any)); assertTrue(tester.nodeRepository().nodes().list().owner(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any)); } } @Test public void deploy_specific_vespa_version() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); ApplicationId application = ProvisioningTester.applicationId(); SystemState state = prepare(application, tester, 2, 2, 3, 3, defaultResources, "6.91"); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void deploy_specific_vespa_version_and_docker_image() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); ApplicationId application = ProvisioningTester.applicationId(); String dockerImageRepo = "docker.domain.tld/my/image"; SystemState state = prepare(application, tester, 2, 2, 3, 3, false, defaultResources, "6.91"); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void test_deployment_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void test_node_limits() { Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); tester.makeReadyHosts(4, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, NodeResources.unspecified()), new ClusterResources(4, 1, NodeResources.unspecified()))); tester.assertNodes("Initial allocation at (allowable) min with default resources", 2, 1, 1.5, 8, 50, 0.3, app1, cluster1); } @Test public void test_changing_limits() { Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 100, 4)); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .flavors(List.of(hostFlavor)) .build(); tester.makeReadyHosts(31, hostFlavor.resources()).activateTenantHosts(); ApplicationId app1 = ProvisioningTester.applicationId("app1"); ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); // Initial deployment tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), resources(8, 4, 4, 20, 40))); tester.assertNodes("Initial allocation at min", 4, 2, 2, 10, 20, app1, cluster1); // Move window above current allocation tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 4, 21, 40), resources(10, 5, 5, 25, 50))); tester.assertNodes("New allocation at new min", 8, 4, 4, 21, 40, app1, cluster1); // Move window below current allocation tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), resources(6, 3, 3, 15, 25))); tester.assertNodes("Allocation preserving resources within new limits", 6, 2, 3, 8.0/4*21 / (6.0/2), 25, app1, cluster1); // Widening window does not change allocation tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 1, 5, 15), resources(8, 4, 4, 21, 30))); tester.assertNodes("Same allocation", 6, 2, 3, 8.0/4*21 / (6.0/2), 25, app1, cluster1); // Changing limits in opposite directions cause a mixture of min and max tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 10, 30, 10), resources(4, 2, 14, 40, 13))); tester.assertNodes("A mix of min and max", 4, 1, 10, 30, 13, app1, cluster1); // Changing group size tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 10), resources(9, 3, 12, 35, 15))); tester.assertNodes("Groups changed", 9, 3, 8, 35, 15, app1, cluster1); // Stop specifying node resources tester.activate(app1, cluster1, Capacity.from(new ClusterResources(6, 3, NodeResources.unspecified()), new ClusterResources(9, 3, NodeResources.unspecified()))); tester.assertNodes("No change", 9, 3, 8, 35, 15, app1, cluster1); } @Test(expected = IllegalArgumentException.class) public void prod_deployment_requires_redundancy() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); prepare(application, 1, 2, 3, 3, defaultResources, tester); } @Test public void below_memory_resource_limit() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, new NodeResources(2, 2, 10, 2), tester); } catch (IllegalArgumentException e) { assertEquals("container cluster 'container0': Min memoryGb size is 2.00 Gb but must be at least 4.00 Gb", e.getMessage()); } } @Test public void below_vcpu_resource_limit() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); try { prepare(application, 2, 2, 3, 3, new NodeResources(0.4, 4, 10, 2), tester); } catch (IllegalArgumentException e) { assertEquals("container cluster 'container0': Min vcpu size is 0.40 but must be at least 0.50", e.getMessage()); } } /** Dev always uses the zone default flavor */ @Test public void dev_deployment_flavor() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, new NodeResources(2, 4, 10, 2)); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); ApplicationId application = ProvisioningTester.applicationId(); SystemState state = prepare(application, 2, 2, 3, 3, new NodeResources(2, 4, 10, 2), tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } /** Test always uses the zone default resources */ @Test public void test_deployment_resources() { NodeResources large = new NodeResources(2, 4, 10, 0.3); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(4, large).activateTenantHosts(); SystemState state = prepare(application, 2, 2, 3, 3, large, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void staging_deployment_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.applicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); // Simulate expiry tester.deactivate(application); try { tester.activate(application, state.allHosts); fail("Expected exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().startsWith("Activation of " + application + " failed")); } } @Test public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(9, defaultResources).activateTenantHosts(); // need 2+2+3+3=10 ApplicationId application = ProvisioningTester.applicationId(); try { prepare(application, 2, 2, 3, 3, defaultResources, tester); fail("Expected exception"); } catch (OutOfCapacityException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); } } @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.applicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified()), false, false)); // No exception; Success } @Test public void out_of_capacity_all_nodes_want_to_retire() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); // Flag all nodes for retirement List readyNodes = tester.makeReadyNodes(5, defaultResources); tester.patchNodes(readyNodes, (node) -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); try { prepare(application, 2, 0, 2, 0, defaultResources, tester); fail("Expected exception"); } catch (OutOfCapacityException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); } } @Test public void want_to_retire_but_cannot_fail() { Capacity capacity = Capacity.from(new ClusterResources(5, 1, defaultResources), false, true); Capacity capacityFORCED = Capacity.from(new ClusterResources(5, 1, defaultResources), false, false); ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); // Create 10 nodes tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacity)); assertEquals(5, tester.nodeRepository().nodes().list(Node.State.active).owner(application).not().retired().size()); assertEquals(0, tester.nodeRepository().nodes().list(Node.State.active).owner(application).retired().size()); // Mark the nodes as want to retire tester.nodeRepository().nodes().list(Node.State.active).owner(application).forEach(node -> tester.patchNode(node, (n) -> n.withWantToRetire(true, Agent.system, tester.clock().instant()))); // redeploy without allow failing tester.activate(application, tester.prepare(application, cluster, capacityFORCED)); // Nodes are not retired since that is unsafe when we cannot fail assertEquals(5, tester.nodeRepository().nodes().list(Node.State.active).owner(application).not().retired().size()); assertEquals(0, tester.nodeRepository().nodes().list(Node.State.active).owner(application).retired().size()); // ... but we still want to tester.nodeRepository().nodes().list(Node.State.active).owner(application).forEach(node -> assertTrue(node.status().wantToRetire())); // redeploy with allowing failing tester.activate(application, tester.prepare(application, cluster, capacity)); // ... old nodes are now retired assertEquals(5, tester.nodeRepository().nodes().list(Node.State.active).owner(application).not().retired().size()); assertEquals(5, tester.nodeRepository().nodes().list(Node.State.active).owner(application).retired().size()); } @Test public void retired_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.applicationId(); Capacity capacityCanFail = Capacity.from(new ClusterResources(5, 1, defaultResources), false, true); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacityCanFail)); assertEquals(0, tester.nodeRepository().nodes().list(Node.State.active).owner(application).retired().size()); tester.patchNode(tester.nodeRepository().nodes().list().owner(application).first().orElseThrow(), n -> n.withWantToRetire(true, Agent.system, tester.clock().instant())); tester.activate(application, tester.prepare(application, cluster, capacityCanFail)); assertEquals(1, tester.nodeRepository().nodes().list(Node.State.active).owner(application).retired().size()); assertEquals(6, tester.nodeRepository().nodes().list(Node.State.active).owner(application).size()); Capacity capacityCannotFail = Capacity.from(new ClusterResources(5, 1, defaultResources), false, false); tester.activate(application, tester.prepare(application, cluster, capacityCannotFail)); assertEquals(1, tester.nodeRepository().nodes().list(Node.State.active).owner(application).retired().size()); assertEquals(6, tester.nodeRepository().nodes().list(Node.State.active).owner(application).size()); } @Test public void highest_node_indexes_are_retired_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application1 = ProvisioningTester.applicationId(); tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); // deploy SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester); tester.activate(application1, state1.allHosts); // decrease cluster sizes SystemState state2 = prepare(application1, 2, 2, 2, 2, defaultResources, tester); tester.activate(application1, state2.allHosts); // content0 assertFalse(state2.hostByMembership("content0", 0, 0).membership().get().retired()); assertFalse(state2.hostByMembership("content0", 0, 1).membership().get().retired()); assertTrue( state2.hostByMembership("content0", 0, 2).membership().get().retired()); assertTrue( state2.hostByMembership("content0", 0, 3).membership().get().retired()); // content1 assertFalse(state2.hostByMembership("content1", 0, 0).membership().get().retired()); assertFalse(state2.hostByMembership("content1", 0, 1).membership().get().retired()); assertTrue( state2.hostByMembership("content1", 0, 2).membership().get().retired()); assertTrue( state2.hostByMembership("content1", 0, 3).membership().get().retired()); } @Test public void node_on_spare_host_retired_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))) .spareCount(1).build(); tester.makeReadyHosts(7, defaultResources).activateTenantHosts(); ApplicationId application = ProvisioningTester.applicationId(); ClusterSpec spec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7.1.2").build(); tester.deploy(application, spec, Capacity.from(new ClusterResources(6, 1, defaultResources))); // Pick out a random application node and make it's parent larger, this will make it the spare host NodeList nodes = tester.nodeRepository().nodes().list(); Node randomNode = nodes.owner(application).shuffle(new Random()).first().get(); tester.nodeRepository().nodes().write(nodes.parentOf(randomNode).get() .with(new Flavor(new NodeResources(2, 10, 20, 8)), Agent.system, tester.nodeRepository().clock().instant()), () -> {}); // Re-deploy application with 1 node less, the retired node should be on the spare host tester.deploy(application, spec, Capacity.from(new ClusterResources(5, 1, defaultResources))); assertTrue(tester.nodeRepository().nodes().node(randomNode.hostname()).get().allocation().get().membership().retired()); } @Test public void application_deployment_retires_nodes_that_want_to_retire() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(10, defaultResources).activateTenantHosts(); // Deploy application { SystemState state = prepare(application, 2, 0, 2, 0,defaultResources, tester); tester.activate(application, state.allHosts); assertEquals(4, tester.getNodes(application, Node.State.active).size()); } // Retire some nodes and redeploy { List nodesToRetire = tester.getNodes(application, Node.State.active).asList().subList(0, 2); tester.patchNodes(nodesToRetire, (node) -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester); tester.activate(application, state.allHosts); List retiredNodes = tester.getNodes(application).retired().asList(); assertEquals(2, retiredNodes.size()); assertTrue("Nodes are retired by system", retiredNodes.stream().allMatch(retiredBy(Agent.system))); } } @Test public void allow_unretire_nodes_allocated_through_type_spec() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId tenantHostAppId = ProvisioningTester.applicationId(); tester.makeReadyHosts(10, defaultResources).prepareAndActivateInfraApplication(tenantHostAppId, NodeType.host); NodeList list = tester.nodeRepository().nodes().list(); assertEquals(10, list.state(Node.State.active).nodeType(NodeType.host).size()); // Pick out 5 random nodes and retire those Set retiredHostnames = list.shuffle(new Random()).first(5).hostnames(); tester.patchNodes(node -> retiredHostnames.contains(node.hostname()), node -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); tester.prepareAndActivateInfraApplication(tenantHostAppId, NodeType.host); assertEquals(retiredHostnames, tester.nodeRepository().nodes().list().retired().hostnames()); Set unretiredHostnames = retiredHostnames.stream().limit(2).collect(Collectors.toSet()); tester.patchNodes(node -> unretiredHostnames.contains(node.hostname()), node -> node.withWantToRetire(false, Agent.system, tester.clock().instant())); tester.prepareAndActivateInfraApplication(tenantHostAppId, NodeType.host); assertEquals(3, tester.nodeRepository().nodes().list().retired().size()); } @Test public void application_deployment_extends_existing_reservations_on_deploy() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); // Deploy fails with out of capacity try { prepare(application, 2, 0, 2, 0, defaultResources, tester); fail("Expected exception"); } catch (OutOfCapacityException ignored) {} assertEquals("Reserved a subset of required nodes", 2, tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); SystemState state = prepare(application, 2, 0, 2, 0, defaultResources, tester); List reserved = tester.getNodes(application, Node.State.reserved).asList(); assertEquals("Reserved required nodes", 4, reserved.size()); assertTrue("Time of event is updated for all nodes", reserved.stream() .allMatch(n -> n.history() .event(History.Event.Type.reserved) .get().at() .equals(tester.clock().instant().truncatedTo(MILLIS)))); // Over 10 minutes pass since first reservation. First set of reserved nodes are not expired tester.clock().advance(Duration.ofMinutes(8).plus(Duration.ofSeconds(1))); ReservationExpirer expirer = new ReservationExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()); expirer.run(); assertEquals("Nodes remain reserved", 4, tester.getNodes(application, Node.State.reserved).size()); tester.activate(application, state.allHosts); assertEquals(4, tester.getNodes(application, Node.State.active).size()); } @Test public void required_capacity_respects_prod_redundancy_requirement() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); try { prepare(application, tester, 1, 0, 1, 0, true, defaultResources, "6.42"); fail("Expected exception"); } catch (IllegalArgumentException ignored) {} } @Test public void devsystem_application_deployment_on_devhost() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.dev, Environment.dev, RegionName.from("no-central"))).build(); tester.makeReadyNodes(4, defaultResources, NodeType.devhost, 1); tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.devhost); ApplicationId application = ProvisioningTester.applicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } @Test public void allocates_reserved_nodes_for_type_spec_deployment() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); Function> prepareAndActivate = app -> tester.activate(app.getApplicationId(), tester.prepare(app.getApplicationId(), app.getClusterSpecWithVersion(Version.fromString("1.2.3")), app.getCapacity())); // Add 2 config server hosts and 2 config servers Flavor flavor = tester.nodeRepository().flavors().getFlavorOrThrow("default"); List nodes = List.of( Node.create("cfghost1", new IP.Config(Set.of("::1:0"), Set.of("::1:1")), "cfghost1", flavor, NodeType.confighost).build(), Node.create("cfghost2", new IP.Config(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(IP.Config.of(Set.of("::2:0"), Set.of("::2:1"), List.of())).build(), Node.create("cfg1", new IP.Config(Set.of("::1:1"), Set.of()), "cfg1", flavor, NodeType.config).parentHostname("cfghost1").build(), Node.create("cfg2", new IP.Config(Set.of("::2:1"), Set.of()), "cfg2", flavor, NodeType.config).parentHostname("cfghost2").build()); tester.nodeRepository().nodes().setReady(tester.nodeRepository().nodes().addNodes(nodes, Agent.system), Agent.system, ProvisioningTest.class.getSimpleName()); InfraApplication cfgHostApp = new ConfigServerHostApplication(); InfraApplication cfgApp = new ConfigServerApplication(); // Attempt to prepare & activate cfg, this should fail as cfg hosts are not active try { prepareAndActivate.apply(cfgApp); } catch (ParentHostUnavailableException ignored) { } assertEquals(2, tester.nodeRepository().nodes().list().owner(cfgApp.getApplicationId()).state(Node.State.reserved).size()); prepareAndActivate.apply(cfgHostApp); // After activating cfg hosts, we can activate cfgs and all 4 should become active prepareAndActivate.apply(cfgApp); assertEquals(4, tester.nodeRepository().nodes().list().state(Node.State.active).size()); } @Test public void cluster_spec_update_for_already_reserved_nodes() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); ApplicationId application = ProvisioningTester.applicationId(); String version1 = "6.42"; String version2 = "6.43"; tester.makeReadyNodes(2, defaultResources); prepare(application, tester, 1, 0, 1, 0, true, defaultResources, version1); tester.getNodes(application, Node.State.reserved).forEach(node -> assertEquals(Version.fromString(version1), node.allocation().get().membership().cluster().vespaVersion())); prepare(application, tester, 1, 0, 1, 0, true, defaultResources, version2); tester.getNodes(application, Node.State.reserved).forEach(node -> assertEquals(Version.fromString(version2), node.allocation().get().membership().cluster().vespaVersion())); } @Test public void change_to_and_from_combined_cluster_does_not_change_node_allocation() { var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); var application = ProvisioningTester.applicationId(); tester.makeReadyHosts(4, defaultResources).activateTenantHosts(); // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); var initialNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, defaultResources)))); // Application is redeployed with cluster type combined cluster = ClusterSpec.request(ClusterSpec.Type.combined, ClusterSpec.Id.from("music")) .vespaVersion("1.2.3") .combinedId(Optional.of(ClusterSpec.Id.from("qrs"))) .build(); var newNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, defaultResources)))); assertEquals("Node allocation remains the same", initialNodes, newNodes); assertEquals("Cluster type is updated", Set.of(ClusterSpec.Type.combined), newNodes.stream().map(n -> n.membership().get().cluster().type()).collect(Collectors.toSet())); // Application is redeployed with cluster type content again cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); newNodes = tester.activate(application, tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, defaultResources)))); assertEquals("Node allocation remains the same", initialNodes, newNodes); assertEquals("Cluster type is updated", Set.of(ClusterSpec.Type.content), newNodes.stream().map(n -> n.membership().get().cluster().type()).collect(Collectors.toSet())); } @Test public void transitions_directly_to_dirty_in_cd() { ApplicationId application = ProvisioningTester.applicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); Capacity capacity = Capacity.from(new ClusterResources(2, 1, defaultResources)); BiConsumer stateAsserter = (zone, state) -> { ProvisioningTester tester = new ProvisioningTester.Builder().zone(zone).build(); tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); tester.activate(application, tester.prepare(application, cluster, capacity)); tester.activate(application, List.of()); assertEquals(2, tester.getNodes(application, state).size()); }; stateAsserter.accept(new Zone(Environment.prod, RegionName.from("us-east")), Node.State.inactive); stateAsserter.accept(new Zone(SystemName.cd, Environment.prod, RegionName.from("us-east")), Node.State.dirty); } private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources flavor, ProvisioningTester tester) { return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42"); } private SystemState prepare(ApplicationId application, ProvisioningTester tester, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources nodeResources, String wantedVersion) { return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, false, nodeResources, wantedVersion); } private SystemState prepare(ApplicationId application, ProvisioningTester tester, int container0Size, int container1Size, int content0Size, int content1Size, boolean required, NodeResources nodeResources, String wantedVersion) { // "deploy prepare" with a two container clusters and a storage cluster having of two groups ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0")).vespaVersion(wantedVersion).build(); ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1")).vespaVersion(wantedVersion).build(); ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0")).vespaVersion(wantedVersion).build(); ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion(wantedVersion).build(); Set container0 = prepare(application, tester, containerCluster0, container0Size, 1, required, nodeResources); Set container1 = prepare(application, tester, containerCluster1, container1Size, 1, required, nodeResources); Set content0 = prepare(application, tester, contentCluster0, content0Size, 1, required, nodeResources); Set content1 = prepare(application, tester, contentCluster1, content1Size, 1, required, nodeResources); Set allHosts = new HashSet<>(); allHosts.addAll(container0); allHosts.addAll(container1); allHosts.addAll(content0); allHosts.addAll(content1); Function capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified()), required, true); int expectedContainer0Size = tester.capacityPolicies().decideSize(container0Size, capacity.apply(container0Size), containerCluster0, application); int expectedContainer1Size = tester.capacityPolicies().decideSize(container1Size, capacity.apply(container1Size), containerCluster1, application); int expectedContent0Size = tester.capacityPolicies().decideSize(content0Size, capacity.apply(content0Size), contentCluster0, application); int expectedContent1Size = tester.capacityPolicies().decideSize(content1Size, capacity.apply(content1Size), contentCluster1, application); assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct", expectedContainer0Size + expectedContainer1Size + expectedContent0Size + expectedContent1Size, tester.nonRetired(allHosts).size()); // Check cluster/group sizes assertEquals(expectedContainer0Size, tester.nonRetired(container0).size()); assertEquals(expectedContainer1Size, tester.nonRetired(container1).size()); assertEquals(expectedContent0Size, tester.nonRetired(content0).size()); assertEquals(expectedContent1Size, tester.nonRetired(content1).size()); // Check cluster membership tester.assertMembersOf(containerCluster0, container0); tester.assertMembersOf(containerCluster1, container1); tester.assertMembersOf(contentCluster0, content0); tester.assertMembersOf(contentCluster1, content1); return new SystemState(allHosts, container0, container1, content0, content1); } private Set prepare(ApplicationId application, ProvisioningTester tester, ClusterSpec cluster, int nodeCount, int groups, boolean required, NodeResources nodeResources) { if (nodeCount == 0) return Set.of(); // this is a shady practice return new HashSet<>(tester.prepare(application, cluster, nodeCount, groups, required, nodeResources)); } private static class SystemState { private Set allHosts; private Set container0; private Set container1; private Set content0; private Set content1; public SystemState(Set allHosts, Set container1, Set container2, Set content0, Set content1) { this.allHosts = allHosts; this.container0 = container1; this.container1 = container2; this.content0 = content0; this.content1 = content1; } /** Returns a host by cluster name and index, or null if there is no host with the given values in this */ public HostSpec hostByMembership(String clusterId, int group, int index) { for (HostSpec host : allHosts) { if ( ! host.membership().isPresent()) continue; ClusterMembership membership = host.membership().get(); if (membership.cluster().id().value().equals(clusterId) && groupMatches(membership.cluster().group(), group) && membership.index() == index) return host; } return null; } private boolean groupMatches(Optional clusterGroup, int group) { if ( ! clusterGroup.isPresent()) return group==0; return clusterGroup.get().index() == group; } public Set hostNames() { return allHosts.stream().map(HostSpec::hostname).collect(Collectors.toSet()); } public HostSpec removeHost(String hostname) { for (Iterator i = allHosts.iterator(); i.hasNext();) { HostSpec host = i.next(); if (host.hostname().equals(hostname)) { i.remove(); return host; } } return null; } public void assertExtends(SystemState other) { assertTrue(this.allHosts.containsAll(other.allHosts)); assertExtends(this.container0, other.container0); assertExtends(this.container1, other.container1); assertExtends(this.content0, other.content0); assertExtends(this.content1, other.content1); } private void assertExtends(Set extension, Set original) { for (HostSpec originalHost : original) { HostSpec newHost = findHost(originalHost.hostname(), extension); org.junit.Assert.assertEquals(newHost.membership(), originalHost.membership()); } } private HostSpec findHost(String hostName, Set hosts) { for (HostSpec host : hosts) if (host.hostname().equals(hostName)) return host; return null; } public void assertEquals(SystemState other) { org.junit.Assert.assertEquals(this.allHosts, other.allHosts); org.junit.Assert.assertEquals(this.container0, other.container0); org.junit.Assert.assertEquals(this.container1, other.container1); org.junit.Assert.assertEquals(this.content0, other.content0); org.junit.Assert.assertEquals(this.content1, other.content1); } } /** A predicate that returns whether a node has been retired by the given agent */ private static Predicate retiredBy(Agent agent) { return (node) -> node.history().event(History.Event.Type.retired) .filter(e -> e.type() == History.Event.Type.retired) .filter(e -> e.agent() == agent) .isPresent(); } private ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); } }