From bcaf74cc7cddd26f315ea9c60ceb8a5f9b665168 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 27 Mar 2020 12:57:10 +0100 Subject: Maintain application min, max and target resources --- .../src/main/java/com/yahoo/config/provision/ClusterResources.java | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'config-provisioning') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java index 11873bc908c..48a201f4f65 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java @@ -44,6 +44,13 @@ public class ClusterResources { return false; } + /** Returns true if this is within the given limits (inclusive) */ + public boolean isWithin(ClusterResources min, ClusterResources max) { + if (this.smallerThan(min)) return false; + if (max.smallerThan(this)) return false; + return true; + } + @Override public boolean equals(Object o) { if (o == this) return true; -- cgit v1.2.3 From 3124be24e2419dbd17ae448b4cd8203e22278fea Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sat, 28 Mar 2020 13:32:18 +0100 Subject: Respect node resource limits --- .../com/yahoo/config/provision/NodeResources.java | 10 +- .../hosted/provision/applications/Cluster.java | 36 ++++--- .../hosted/provision/autoscale/Autoscaler.java | 8 +- .../vespa/hosted/provision/autoscale/Resource.java | 8 +- .../provision/autoscale/ResourceIterator.java | 63 +++++++++--- .../maintenance/AutoscalingMaintainer.java | 6 +- .../provision/autoscale/AutoscalingTest.java | 89 +++++++++++++++-- .../provision/autoscale/AutoscalingTester.java | 13 +++ .../maintenance/AutoscalingMaintainerTest.java | 110 +++++++++++++++++++++ .../provision/maintenance/MaintenanceTester.java | 3 + .../provision/maintenance/RebalancerTest.java | 16 +-- .../provision/provisioning/ProvisioningTester.java | 29 +++++- 12 files changed, 331 insertions(+), 60 deletions(-) create mode 100644 node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java (limited to 'config-provisioning') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 6d7fe752e46..5fc05a87a7d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -114,26 +114,32 @@ public class NodeResources { public StorageType storageType() { return storageType; } public NodeResources withVcpu(double vcpu) { + if (vcpu == this.vcpu) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withMemoryGb(double memoryGb) { + if (memoryGb == this.memoryGb) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withDiskGb(double diskGb) { + if (diskGb == this.diskGb) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources withBandwidthGbps(double bandwidthGbps) { + if (bandwidthGbps == this.bandwidthGbps) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } - public NodeResources with(DiskSpeed speed) { - return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType); + public NodeResources with(DiskSpeed diskSpeed) { + if (diskSpeed == this.diskSpeed) return this; + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } public NodeResources with(StorageType storageType) { + if (storageType == this.storageType) return this; return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 6ff827ac92b..6ff7f41be8f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.applications; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.NodeResources; import java.util.Objects; import java.util.Optional; @@ -15,38 +16,51 @@ import java.util.Optional; */ public class Cluster { - private final ClusterResources minResources, maxResources; - private final Optional targetResources; + private final ClusterResources min, max; + private final Optional target; Cluster(ClusterResources minResources, ClusterResources maxResources, Optional targetResources) { - this.minResources = Objects.requireNonNull(minResources); - this.maxResources = Objects.requireNonNull(maxResources); + this.min = Objects.requireNonNull(minResources); + this.max = Objects.requireNonNull(maxResources); Objects.requireNonNull(targetResources); if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources)) - this.targetResources = Optional.empty(); + this.target = Optional.empty(); else - this.targetResources = targetResources; + this.target = targetResources; } /** Returns the configured minimal resources in this cluster */ - public ClusterResources minResources() { return minResources; } + public ClusterResources minResources() { return min; } /** Returns the configured maximal resources in this cluster */ - public ClusterResources maxResources() { return maxResources; } + public ClusterResources maxResources() { return max; } /** * Returns the computed resources (between min and max, inclusive) this cluster should * have allocated at the moment, or empty if the system currently have no opinion on this. */ - public Optional targetResources() { return targetResources; } + public Optional targetResources() { return target; } public Cluster withTarget(ClusterResources target) { - return new Cluster(minResources, maxResources, Optional.of(target)); + return new Cluster(min, max, Optional.of(target)); } public Cluster withoutTarget() { - return new Cluster(minResources, maxResources, Optional.empty()); + return new Cluster(min, max, Optional.empty()); + } + + public NodeResources capAtLimits(NodeResources resources) { + resources = resources.withVcpu(between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), resources.vcpu())); + resources = resources.withMemoryGb(between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), resources.memoryGb())); + resources = resources.withDiskGb(between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), resources.diskGb())); + return resources; + } + + private double between(double min, double max, double value) { + value = Math.max(min, value); + value = Math.min(max, value); + return value; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 6612525685a..dc873ce4e69 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -84,7 +84,8 @@ public class Autoscaler { Optional bestAllocation = findBestAllocation(cpuLoad.get(), memoryLoad.get(), diskLoad.get(), - currentAllocation); + currentAllocation, + cluster); if (bestAllocation.isEmpty()) return Optional.empty(); if (closeToIdeal(Resource.cpu, cpuLoad.get()) && @@ -97,9 +98,10 @@ public class Autoscaler { } private Optional findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, - AllocatableClusterResources currentAllocation) { + AllocatableClusterResources currentAllocation, + Cluster cluster) { Optional bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation); i.hasNext(); ) { + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster); i.hasNext(); ) { ClusterResources allocation = i.next(); Optional allocatableResources = toAllocatableResources(allocation, currentAllocation.clusterType()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java index e84544e7e7b..3d5ce8881e0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java @@ -12,7 +12,7 @@ public enum Resource { /** Cpu utilization ratio */ cpu { - String metricName() { return "cpu.util"; } + public String metricName() { return "cpu.util"; } double idealAverageLoad() { return 0.2; } double valueFrom(NodeResources resources) { return resources.vcpu(); } double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio @@ -20,7 +20,7 @@ public enum Resource { /** Memory utilization ratio */ memory { - String metricName() { return "mem_total.util"; } + public String metricName() { return "mem_total.util"; } double idealAverageLoad() { return 0.7; } double valueFrom(NodeResources resources) { return resources.memoryGb(); } double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio @@ -28,13 +28,13 @@ public enum Resource { /** Disk utilization ratio */ disk { - String metricName() { return "disk.util"; } + public String metricName() { return "disk.util"; } double idealAverageLoad() { return 0.6; } double valueFrom(NodeResources resources) { return resources.diskGb(); } double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio }; - abstract String metricName(); + public abstract String metricName(); /** The load we should have of this resource on average, when one node in the cluster is down */ abstract double idealAverageLoad(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index 82c07345c7f..19909e40441 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.applications.Cluster; /** * Provides iteration over possible cluster resource allocations given a target total load @@ -10,16 +11,19 @@ import com.yahoo.config.provision.NodeResources; */ public class ResourceIterator { - // Configured min and max nodes TODO: These should come from the application package - private static final int minimumNodesPerCluster = 3; // Since this is with redundancy it cannot be lower than 2 - private static final int maximumNodesPerCluster = 150; + // Configured min and max nodes for suggestions for apps which have not activated autoscaling + private static final int minimumNodes = 3; // Since this is with redundancy it cannot be lower than 2 + private static final int maximumNodes = 150; // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component // proportional to document count. We must account for this when comparing configurations with more or fewer nodes. // TODO: Measure this, and only take it into account with queries private static final double fixedCpuCostFraction = 0.1; - // Describes the observed state + // Prescribed state + private final Cluster cluster; + + // Observed state private final AllocatableClusterResources allocation; private final double cpuLoad; private final double memoryLoad; @@ -33,7 +37,9 @@ public class ResourceIterator { // Iterator state private int currentNodes; - public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation) { + public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, + AllocatableClusterResources currentAllocation, + Cluster cluster) { this.cpuLoad = cpuLoad; this.memoryLoad = memoryLoad; this.diskLoad = diskLoad; @@ -42,6 +48,8 @@ public class ResourceIterator { groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); allocation = currentAllocation; + this.cluster = cluster; + // What number of nodes is it effective to add or remove at the time from this cluster? // This is the group size, since we (for now) assume the group size is decided by someone wiser than us // and we decide the number of groups. @@ -49,30 +57,53 @@ public class ResourceIterator { singleGroupMode = currentAllocation.groups() == 1; nodeIncrement = singleGroupMode ? 1 : groupSize; + // Step down to the right starting point currentNodes = currentAllocation.nodes(); - while (currentNodes - nodeIncrement >= minimumNodesPerCluster - && (singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy + while (currentNodes - nodeIncrement >= minNodes() + && ( singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy currentNodes -= nodeIncrement; } + /** If autoscaling is not enabled (meaning max and min resources are the same), we want to suggest */ + private boolean suggestMode() { + return cluster.minResources().equals(cluster.maxResources()); + } + public ClusterResources next() { - int nodesWithRedundancy = currentNodes - (singleGroupMode ? 1 : groupSize); - ClusterResources next = new ClusterResources(currentNodes, - singleGroupMode ? 1 : currentNodes / groupSize, - resourcesFor(nodesWithRedundancy)); + ClusterResources next = resourcesWith(currentNodes); currentNodes += nodeIncrement; + System.out.println("Candidate: " + next); return next; } public boolean hasNext() { - return currentNodes <= maximumNodesPerCluster; + return currentNodes <= maxNodes(); + } + + private int minNodes() { + if (suggestMode()) return minimumNodes; + if (singleGroupMode) return cluster.minResources().nodes(); + return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize ); + } + + private int maxNodes() { + if (suggestMode()) return maximumNodes; + if (singleGroupMode) return cluster.maxResources().nodes(); + return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize ); + } + + private ClusterResources resourcesWith(int nodes) { + int nodesWithRedundancy = nodes - (singleGroupMode ? 1 : groupSize); + return new ClusterResources(nodes, + singleGroupMode ? 1 : nodes / groupSize, + nodeResourcesWith(nodesWithRedundancy)); } /** * For the observed load this instance is initialized with, returns the resources needed per node to be at * ideal load given a target node count */ - private NodeResources resourcesFor(int nodeCount) { + private NodeResources nodeResourcesWith(int nodeCount) { // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size) // Memory and disk: Scales with group size @@ -103,7 +134,11 @@ public class ResourceIterator { disk = nodeUsage(Resource.disk, diskLoad) / Resource.disk.idealAverageLoad(); } } - return allocation.realResources().withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); + + NodeResources resources = allocation.realResources().withVcpu(cpu).withMemoryGb(memory).withDiskGb(disk); + if ( ! suggestMode()) + resources = cluster.capAtLimits(resources); + return resources; } private double clusterUsage(Resource resource, double load) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 064c7db5e60..7073ab5d1a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -95,7 +95,7 @@ public class AutoscalingMaintainer extends Maintainer { int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - log.info("Scaling suggestion: " + application + " " + clusterType + " " + clusterId + + log.info("Scaling suggestion for " + application + " " + clusterType + " " + clusterId + ":" + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + "\nto " + toString(target.nodes(), target.groups(), target.advertisedResources())); lastLogged.put(new Pair<>(application, clusterId), nodeRepository().clock().instant()); @@ -103,8 +103,8 @@ public class AutoscalingMaintainer extends Maintainer { private String toString(int nodes, int groups, NodeResources resources) { return String.format(nodes + (groups > 1 ? " (in " + groups + " groups)" : "") + - " * [vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb]" + - " (total: [vcpu: %4$.1f, memory: %5$.1f Gb, disk: %6$.1f Gb])," + + " * [vcpu: %0$.1f, memory: %1$.1f Gb, disk %2$.1f Gb]" + + " (total: [vcpu: %3$.1f, memory: %4$.1f Gb, disk: %5$.1f Gb])", resources.vcpu(), resources.memoryGb(), resources.diskGb(), nodes * resources.vcpu(), nodes * resources.memoryGb(), nodes * resources.diskGb()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index f3b6606a970..0e6d2365490 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -64,8 +64,8 @@ public class AutoscalingTest { assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1, cluster1.id(), min, max)); tester.addMeasurements(Resource.cpu, 0.1f, 1f, 120, application1); - tester.assertResources("Scaling down since resource usage has gone down significantly", - 26, 1, 0.6, 16.0, 16.0, + tester.assertResources("Scaling down to minimum since usage has gone down significantly", + 14, 1, 1.0, 30.8, 30.8, tester.autoscale(application1, cluster1.id(), min, max)); } @@ -98,10 +98,87 @@ public class AutoscalingTest { } @Test - public void testAutoscalingGroupSize1() { + public void testAutoscalingRespectsUpperLimit() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); + ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 120, application1); + tester.addMeasurements(Resource.memory, 0.95f, 120, application1); + tester.addMeasurements(Resource.disk, 0.95f, 120, application1); + tester.assertResources("Scaling up to limit since resource usage is too high", + 6, 1, 2.4, 78.0, 79.0, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void testAutoscalingRespectsLowerLimit() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1.8, 7.4, 8.5, 1)); + ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.05f, 120, application1); + tester.addMeasurements(Resource.memory, 0.05f, 120, application1); + tester.addMeasurements(Resource.disk, 0.05f, 120, application1); + tester.assertResources("Scaling down to limit since resource usage is low", + 3, 1, 1.8, 7.4, 8.5, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void testAutoscalingRespectsGroupLimit() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 5, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + tester.assertResources("Scaling up since resource usage is too high", + 6, 6, 2.5, 80.0, 80.0, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + /** This condition ensures we get recommendation suggestions when deactivated */ + @Test + public void testAutoscalingLimitsAreIgnoredIfMinEqualsMax() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = min; + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + tester.assertResources("Scaling up since resource usage is too high", + 7, 1, 2.6, 80.0, 80.0, + tester.autoscale(application1, cluster1.id(), min, max)); + } + + @Test + public void testAutoscalingGroupSize1() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); @@ -118,8 +195,8 @@ public class AutoscalingTest { @Test public void testAutoscalingGroupSize3() { NodeResources resources = new NodeResources(3, 100, 100, 1); - ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); + ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); AutoscalingTester tester = new AutoscalingTester(resources); ApplicationId application1 = tester.applicationId("application1"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 8043d5b2d39..ebc4d158ded 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -141,6 +141,19 @@ class AutoscalingTester { } } + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId) { + List nodes = nodeRepository().getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + clock().advance(Duration.ofMinutes(1)); + for (Node node : nodes) { + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + } + public Optional autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId, true).withClusterLimits(clusterId, min, max); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java new file mode 100644 index 00000000000..413c14d132d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -0,0 +1,110 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.autoscale.Resource; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertTrue; + +/** + * Tests the autoscaling maintainer integration. + * The specific recommendations of the autoscaler are not tested here. + * + * @author bratseth + */ +public class AutoscalingMaintainerTest { + + @Test + public void testAutoscalingMaintainer() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(); + + ApplicationId app2 = tester.makeApplicationId("app2"); + ClusterSpec cluster2 = tester.clusterSpec(); + + NodeResources lowResources = new NodeResources(4, 4, 10, 0.1); + NodeResources highResources = new NodeResources(6.5, 9, 20, 0.1); + + Map apps = Map.of( + app1, new MockDeployer.ApplicationContext(app1, cluster1, Capacity.from(new ClusterResources(2, 1, lowResources))), + app2, new MockDeployer.ApplicationContext(app2, cluster2, Capacity.from(new ClusterResources(2, 1, highResources)))); + MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); + + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + AutoscalingMaintainer maintainer = new AutoscalingMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), + nodeMetricsDb, + deployer, + Duration.ofMinutes(1)); + maintainer.maintain(); // noop + assertTrue(deployer.lastDeployTime(app1).isEmpty()); + assertTrue(deployer.lastDeployTime(app2).isEmpty()); + + tester.makeReadyNodes(20, "flt", NodeType.host, 8); + tester.deployZoneApp(); + + tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, lowResources), false, true)); + tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, lowResources), + new ClusterResources(10, 1, highResources), false, true)); + + maintainer.maintain(); // noop + assertTrue(deployer.lastDeployTime(app1).isEmpty()); + assertTrue(deployer.lastDeployTime(app2).isEmpty()); + + addMeasurements(Resource.cpu, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.cpu, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + + maintainer.maintain(); + assertTrue(deployer.lastDeployTime(app1).isEmpty()); // since autoscaling is off + assertTrue(deployer.lastDeployTime(app2).isPresent()); + } + + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId, + NodeRepository nodeRepository, NodeMetricsDb db) { + List nodes = nodeRepository.getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + for (Node node : nodes) + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + nodeRepository.clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java index 4344016c6fe..664809dc3ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; @@ -14,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import java.time.Instant; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 8c9d5cca54b..387f614c5eb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -55,7 +55,7 @@ public class RebalancerTest { Rebalancer rebalancer = new Rebalancer(deployer, tester.nodeRepository(), - new IdentityHostResourcesCalculator(), + tester.identityHostResourcesCalculator(), Optional.empty(), metric, tester.clock(), @@ -149,18 +149,4 @@ public class RebalancerTest { return b.build(); } - private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { - - @Override - public NodeResources realResourcesOf(Node node) { - return node.flavor().resources(); - } - - @Override - public NodeResources advertisedResourcesOf(Flavor flavor) { - return flavor.resources(); - } - - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 3e7104380a0..a8df47aab1a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -233,6 +233,10 @@ public class ProvisioningTester { InstanceName.from(UUID.randomUUID().toString())); } + public ApplicationId makeApplicationId(String applicationName) { + return ApplicationId.from("tenant", applicationName, "default"); + } + public List makeReadyNodes(int n, String flavor) { return makeReadyNodes(n, flavor, NodeType.tenant); } @@ -418,12 +422,15 @@ public class ProvisioningTester { } public List deploy(ApplicationId application, Capacity capacity) { - List prepared = prepare(application, clusterSpec(), capacity); + return deploy(application, clusterSpec(), capacity); + } + + public List deploy(ApplicationId application, ClusterSpec cluster, Capacity capacity) { + List prepared = prepare(application, cluster, capacity); activate(application, Set.copyOf(prepared)); return getNodes(application, Node.State.active).asList(); } - /** Returns the hosts from the input list which are not retired */ public List nonRetired(Collection hosts) { return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList()); @@ -522,4 +529,22 @@ public class ProvisioningTester { @Override public void log(Level level, String message) { } } + public IdentityHostResourcesCalculator identityHostResourcesCalculator() { + return new IdentityHostResourcesCalculator(); + } + + private static class IdentityHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources realResourcesOf(Node node) { + return node.flavor().resources(); + } + + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { + return flavor.resources(); + } + + } + } -- cgit v1.2.3