diff options
Diffstat (limited to 'node-repository/src/test')
8 files changed, 194 insertions, 27 deletions
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeSkewTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeSkewTest.java new file mode 100644 index 00000000000..85ff4fe4264 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeSkewTest.java @@ -0,0 +1,45 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision; + +import com.yahoo.config.provision.NodeResources; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class NodeSkewTest { + + private static final double d = 0.0001; + + @Test + public void testNodeSkewComputation() { + // No skew + assertEquals(0, Node.skew(r(6, 4, 2), r(6, 4, 2)), d); + assertEquals(0, Node.skew(r(6, 4, 2), r(0, 0, 0)), d); + assertEquals(0, Node.skew(r(6, 4, 2), r(3, 2, 1)), d); + + // Extremely skewed + assertEquals(0.2222, Node.skew(r(6, 4, 2), r(0, 4, 0)), d); + // A little less + assertEquals(0.1666, Node.skew(r(6, 4, 2), r(3, 4, 0)), d); + // A little less + assertEquals(0.0555, Node.skew(r(6, 4, 2), r(3, 4, 1)), d); + // The same, since being at half and full is equally skewed here + assertEquals(0.0555, Node.skew(r(6, 4, 2), r(3, 4, 2)), d); + // Almost not skewed + assertEquals(0.0062, Node.skew(r(6, 4, 2), r(5, 4, 2)), d); + + // Skew is scale free + assertEquals(0.0201, Node.skew(r( 6, 4, 2), r(1, 1, 1)), d); + // - all dimensions twice as large + assertEquals(0.0201, Node.skew(r(12, 8, 4), r(2, 2, 2)), d); + // - just one dimension twice as large + assertEquals(0.0201, Node.skew(r(12, 4, 2), r(2, 1, 1)), d); + } + + private NodeResources r(double vcpu, double memGb, double diskGb) { + return new NodeResources(vcpu, memGb, diskGb, 1); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index dc8650c122e..2be9580f78d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -61,6 +61,7 @@ import static org.mockito.Mockito.when; * @author freva */ public class DynamicProvisioningMaintainerTest { + private final HostProvisionerTester tester = new HostProvisionerTester(); private final HostProvisioner hostProvisioner = mock(HostProvisioner.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 2383f4529a7..d409927701c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -138,7 +138,7 @@ public class InactiveAndFailedExpirerTest { // Flag one node for retirement and redeploy { Node toRetire = tester.getNodes(applicationId, Node.State.active).asList().get(0); - tester.patchNode(toRetire.with(toRetire.status().withWantToRetire(true))); + tester.patchNode(toRetire.withWantToRetire(true, Agent.operator, tester.clock().instant())); List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1); tester.activate(applicationId, new HashSet<>(hostSpecs)); } @@ -160,7 +160,7 @@ public class InactiveAndFailedExpirerTest { Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any()); new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30), - Duration.ofMinutes(10)).run(); + Duration.ofMinutes(10)).run(); assertEquals(1, tester.nodeRepository().getNodes(Node.State.inactive).size()); // Inactive times out and one node is moved to parked diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 798df66f6cd..afada3c6f0f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -66,4 +66,5 @@ public class MaintenanceTester { .map(n -> n.withCurrentRebootGeneration(n.status().reboot().wanted(), Instant.now(clock))) .collect(Collectors.toList()); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java new file mode 100644 index 00000000000..2325bf3118a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -0,0 +1,115 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.time.Duration; +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class RebalancerTest { + + @Test + public void testRebalancing() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); + Rebalancer rebalancer = new Rebalancer(tester.nodeRepository(), + new IdentityHostResourcesCalculator(), + tester.clock(), + Duration.ofMinutes(1)); + + NodeResources cpuResources = new NodeResources(8, 4, 1, 0.1); + NodeResources memResources = new NodeResources(4, 9, 1, 0.1); + + tester.makeReadyNodes(3, "flt", NodeType.host, 8); + tester.deployZoneApp(tester); + + // Cpu heavy application - causing 1 of these nodes to be skewed + ApplicationId cpuApp = makeApplicationId("t1", "a1"); + deployApp(cpuApp, clusterSpec("c"), cpuResources, tester, 1); + String cpuSkewedNodeHostname = tester.nodeRepository().getNodes(cpuApp).get(0).hostname(); + + rebalancer.maintain(); + assertFalse("No better place to move the skewed node, so no action is taken", + tester.nodeRepository().getNode(cpuSkewedNodeHostname).get().status().wantToRetire()); + + tester.makeReadyNodes(1, "cpu", NodeType.host, 8); + + rebalancer.maintain(); + assertTrue("We can now move the node to the cpu skewed host to reduce skew", + tester.nodeRepository().getNode(cpuSkewedNodeHostname).get().status().wantToRetire()); + + ApplicationId memApp = makeApplicationId("t2", "a2"); + deployApp(memApp, clusterSpec("c"), memResources, tester, 1); + assertEquals("Assigned to a flat node as that causes least skew", "flt", + tester.nodeRepository().list().parentOf(tester.nodeRepository().getNodes(memApp).get(0)).get().flavor().name()); + String memSkewedNodeHostname = tester.nodeRepository().getNodes(memApp).get(0).hostname(); + + tester.makeReadyNodes(1, "mem", NodeType.host, 8); + rebalancer.maintain(); + assertFalse("The mem skewed node is not set want to retire as the cpu skewed node still is", + tester.nodeRepository().getNode(memSkewedNodeHostname).get().status().wantToRetire()); + + Node cpuSkewedNode = tester.nodeRepository().getNode(cpuSkewedNodeHostname).get(); + tester.nodeRepository().write(cpuSkewedNode.withWantToRetire(false, Agent.system, tester.clock().instant()), + tester.nodeRepository().lock(cpuSkewedNode)); + rebalancer.maintain(); + assertTrue("The mem skewed node is now scheduled for moving", + tester.nodeRepository().getNode(memSkewedNodeHostname).get().status().wantToRetire()); + assertFalse("(The cpu skewed node is not because it causes slightly less skew)", + tester.nodeRepository().getNode(cpuSkewedNodeHostname).get().status().wantToRetire()); + } + + private ClusterSpec clusterSpec(String clusterId) { + return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); + } + + private ApplicationId makeApplicationId(String tenant, String appName) { + return ApplicationId.from(tenant, appName, "default"); + } + + private void deployApp(ApplicationId id, ClusterSpec spec, NodeResources flavor, ProvisioningTester tester, int nodeCount) { + List<HostSpec> hostSpec = tester.prepare(id, spec, nodeCount, 1, flavor); + tester.activate(id, new HashSet<>(hostSpec)); + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + + private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources availableCapacityOf(NodeResources hostResources) { + return hostResources; + } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java index 4037bc52064..1318a6105f8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java @@ -64,7 +64,7 @@ public class DynamicDockerAllocationTest { public void relocate_nodes_from_spare_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(4, "host-small", NodeType.host, 32); - deployZoneApp(tester); + tester.deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources flavor = new NodeResources(1, 4, 10, 1); @@ -107,7 +107,7 @@ public class DynamicDockerAllocationTest { public void relocate_failed_nodes() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - deployZoneApp(tester); + tester.deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources resources = new NodeResources(1, 4, 10, 0.3); @@ -156,7 +156,7 @@ public class DynamicDockerAllocationTest { tester.makeReadyNodes(3, "flt", NodeType.host, 8); // cpu: 30, mem: 30 tester.makeReadyNodes(3, "cpu", NodeType.host, 8); // cpu: 40, mem: 20 tester.makeReadyNodes(3, "mem", NodeType.host, 8); // cpu: 20, mem: 40 - deployZoneApp(tester); + tester.deployZoneApp(tester); NodeResources fltResources = new NodeResources(6, 6, 1, 0.1); NodeResources cpuResources = new NodeResources(8, 4, 1, 0.1); NodeResources memResources = new NodeResources(4, 8, 1, 0.1); @@ -199,7 +199,7 @@ public class DynamicDockerAllocationTest { public void do_not_relocate_nodes_from_spare_if_no_where_to_relocate_them() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, "host-small", NodeType.host, 32); - deployZoneApp(tester); + tester.deployZoneApp(tester); List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); NodeResources flavor = new NodeResources(1, 4, 10, 1); @@ -226,7 +226,7 @@ public class DynamicDockerAllocationTest { public void multiple_groups_are_on_separate_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - deployZoneApp(tester); + tester.deployZoneApp(tester); //Deploy an application having 6 nodes (3 nodes in 2 groups). We only have 5 docker hosts available ApplicationId application1 = tester.makeApplicationId(); @@ -247,7 +247,7 @@ public class DynamicDockerAllocationTest { // Setup test ApplicationId application1 = tester.makeApplicationId(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); - deployZoneApp(tester); + tester.deployZoneApp(tester); NodeResources flavor = new NodeResources(1, 4, 10, 1); // Deploy initial state (can max deploy 3 nodes due to redundancy requirements) @@ -276,7 +276,7 @@ public class DynamicDockerAllocationTest { public void non_prod_zones_do_not_have_spares() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(3, "host-small", NodeType.host, 32); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application1 = tester.makeApplicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 10, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); @@ -289,7 +289,7 @@ public class DynamicDockerAllocationTest { public void cd_uses_slow_disk_nodes_for_docker_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(SystemName.cd, Environment.test, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(4, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application1 = tester.makeApplicationId(); List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, new NodeResources(1, 4, 10, 1)); tester.activate(application1, ImmutableSet.copyOf(hosts)); @@ -309,7 +309,7 @@ public class DynamicDockerAllocationTest { public void provision_dual_stack_containers() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, "host-large", NodeType.host, 10, true); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); List<HostSpec> hosts = tester.prepare(application, clusterSpec("myContent.t1.a1"), 2, 1, new NodeResources(1, 4, 10, 1)); @@ -340,7 +340,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); @@ -357,7 +357,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); @@ -379,7 +379,7 @@ public class DynamicDockerAllocationTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 12, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); @@ -398,7 +398,7 @@ public class DynamicDockerAllocationTest { public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(5, 20, 140, 3)), NodeType.host, 10, true); - deployZoneApp(tester); + tester.deployZoneApp(tester); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false); @@ -453,18 +453,6 @@ public class DynamicDockerAllocationTest { return b.build(); } - private void deployZoneApp(ProvisioningTester tester) { - ApplicationId applicationId = tester.makeApplicationId(); - List<HostSpec> list = tester.prepare(applicationId, - ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from("node-admin"), - Version.fromString("6.42"), - false), - Capacity.fromRequiredNodeType(NodeType.host), - 1); - tester.activate(applicationId, ImmutableSet.copyOf(list)); - } - private boolean isInactiveOrRetired(Node node) { boolean isInactive = node.state().equals(Node.State.inactive); boolean isRetired = false; @@ -478,4 +466,5 @@ public class DynamicDockerAllocationTest { private ClusterSpec clusterSpec(String clusterId) { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 6c5cc198c67..3ad598b1235 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -397,6 +398,18 @@ public class ProvisioningTester { return nodes; } + public void deployZoneApp(ProvisioningTester tester) { + ApplicationId applicationId = tester.makeApplicationId(); + List<HostSpec> list = tester.prepare(applicationId, + ClusterSpec.request(ClusterSpec.Type.container, + ClusterSpec.Id.from("node-admin"), + Version.fromString("6.42"), + false), + Capacity.fromRequiredNodeType(NodeType.host), + 1); + tester.activate(applicationId, ImmutableSet.copyOf(list)); + } + /** Returns the hosts from the input list which are not retired */ public List<HostSpec> nonRetired(Collection<HostSpec> hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json index cfb39e7e5b1..02746f1c79a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json @@ -40,6 +40,9 @@ "name": "ProvisionedExpirer" }, { + "name": "Rebalancer" + }, + { "name": "ReservationExpirer" }, { |