diff options
11 files changed, 183 insertions, 25 deletions
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java index be57b24d92f..57185d55131 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java @@ -9,9 +9,9 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; /** - * @author Unknowm */ public class MetricsFetcherTest { + private static int port = 9; //port number is not used in this test @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 93d4ad5bc47..0632d85906a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -132,10 +132,11 @@ public class Autoscaler { NodeResources nodeResources = nodeResourceLimits.enlargeToLegal(resources.nodeResources(), cluster.type()); if (allowsHostSharing(nodeRepository.zone().cloud())) { // return the requested resources, or empty if they cannot fit on existing hosts - for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) + for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) { if (flavor.resources().satisfies(nodeResources)) return Optional.of(new AllocatableClusterResources(resources.with(nodeResources), nodeResources)); + } return Optional.empty(); } else { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index a599606c314..653b786ebe5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -47,12 +47,15 @@ public class MetricsResponse { long timestamp = node.field("timestamp").asLong(); Map<String, Double> values = consumeMetrics(node.field("metrics")); for (Resource resource : Resource.values()) - addMetricIfPresent(hostname, resource.metricName(), timestamp, values); + addMetricIfPresent(hostname, resource, timestamp, values); } - private void addMetricIfPresent(String hostname, String metricName, long timestamp, Map<String, Double> values) { - if (values.containsKey(metricName)) - metricValues.add(new NodeMetrics.MetricValue(hostname, metricName, timestamp, values.get(metricName).floatValue())); + private void addMetricIfPresent(String hostname, Resource resource, long timestamp, Map<String, Double> values) { + if (values.containsKey(resource.metricName())) + metricValues.add(new NodeMetrics.MetricValue(hostname, + resource.metricName(), + timestamp, + values.get(resource.metricName()))); } private void consumeServiceMetrics(String hostname, Inspector node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java index 97ac1e72be9..8abc2327d88 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java @@ -24,9 +24,9 @@ public interface NodeMetrics { private final String hostname; private final String name; private long timestamp; - private final float value; + private final double value; - public MetricValue(String hostname, String name, long timestamp, float value) { + public MetricValue(String hostname, String name, long timestamp, double value) { this.hostname = hostname; this.name = name; this.timestamp = timestamp; @@ -36,7 +36,7 @@ public interface NodeMetrics { public String hostname() { return hostname; } public String name() { return name; } public long timestamp() { return timestamp; } - public float value() { return value; } + public double value() { return value; } @Override public String toString() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java index 14a35e3efbc..aa20752d8a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java @@ -11,6 +11,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -22,6 +23,7 @@ import java.util.stream.Stream; */ public class NodeMetricsDb { + private Logger log = Logger.getLogger(NodeMetricsDb.class.getName()); private static final Duration dbWindow = Duration.ofHours(24); /** Measurements by key. Each list of measurements is sorted by increasing timestamp */ @@ -32,18 +34,21 @@ public class NodeMetricsDb { /** Add a measurement to this */ public void add(Collection<NodeMetrics.MetricValue> metricValues) { + log.fine("Adding " + metricValues.size() + " metric values" + + (metricValues.size() > 0 ? ". First: " + metricValues.iterator().next() : "")); synchronized (lock) { for (var value : metricValues) { - List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), - Resource.fromMetric(value.name())), + Resource resource = Resource.fromMetric(value.name()); + List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), resource), (__) -> new ArrayList<>()); - measurements.add(new Measurement(value.timestamp(), value.value())); + measurements.add(new Measurement(value.timestamp(), (float)resource.valueFromMetric(value.value()))); } } } /** Must be called intermittently (as long as add is called) to gc old measurements */ public void gc(Clock clock) { + int gcCount = 0; synchronized (lock) { // TODO: We may need to do something more complicated to avoid spending too much memory to // lower the measurement interval (see NodeRepositoryMaintenance) @@ -53,14 +58,16 @@ public class NodeMetricsDb { long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli(); for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) { List<Measurement> measurements = i.next(); - - while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) + while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp) { measurements.remove(0); + gcCount++; + } if (measurements.isEmpty()) i.remove(); } } + log.fine("Gc'ed " + gcCount + " metric values"); } /** Returns a window within which we can ask for specific information from this db */ @@ -80,6 +87,9 @@ public class NodeMetricsDb { public int measurementCount() { synchronized (lock) { + List<MeasurementKey> matches = keys.stream().filter(key -> db.get(key) != null).collect(Collectors.toList()); + List<MeasurementKey> nonMatches = keys.stream().filter(key -> db.get(key) == null).collect(Collectors.toList()); + log.fine("Counting measurements after " + startTime + ". Matches: " + matches + ". Non-matches: " + nonMatches); return (int) keys.stream() .flatMap(key -> db.getOrDefault(key, List.of()).stream()) .filter(measurement -> measurement.timestamp >= startTime) @@ -149,6 +159,9 @@ public class NodeMetricsDb { return true; } + @Override + public String toString() { return "measurements of " + resource + " for " + hostname; } + } private static class Measurement { @@ -164,6 +177,9 @@ public class NodeMetricsDb { this.value = value; } + @Override + public String toString() { return "measurement at " + timestamp + ": " + value; } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java index 917ac6d3796..e0ecbcfbf3c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java @@ -38,6 +38,7 @@ public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics private final HttpClient httpClient; @Inject + @SuppressWarnings("unused") public NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator) { this(nodeRepository, orchestrator, new ApacheHttpClient()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java index 9c85ca870d5..0eac14d61ca 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java @@ -10,22 +10,28 @@ import com.yahoo.config.provision.NodeResources; */ public enum Resource { + /** Cpu utilization ratio */ cpu { String metricName() { return "cpu.util"; } double idealAverageLoad() { return 0.2; } double valueFrom(NodeResources resources) { return resources.vcpu(); } + double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio }, + /** Memory utilization ratio */ memory { - String metricName() { return "memory.util"; } + String metricName() { return "mem.util"; } double idealAverageLoad() { return 0.7; } double valueFrom(NodeResources resources) { return resources.memoryGb(); } + double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio }, + /** Disk utilization ratio */ disk { String metricName() { return "disk.util"; } double idealAverageLoad() { return 0.7; } double valueFrom(NodeResources resources) { return resources.diskGb(); } + double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio }; abstract String metricName(); @@ -35,6 +41,8 @@ public enum Resource { abstract double valueFrom(NodeResources resources); + abstract double valueFromMetric(double metricValue); + public static Resource fromMetric(String metricName) { for (Resource resource : values()) if (resource.metricName().equals(metricName)) return resource; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java new file mode 100644 index 00000000000..781b48e9561 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -0,0 +1,124 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; +import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; +import org.junit.Test; + +import java.time.Duration; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class AutoscalingIntegrationTest { + + @Test + public void testComponentIntegration() { + NodeResources nodes = new NodeResources(1, 10, 100, 1); + NodeResources hosts = new NodeResources(3, 20, 200, 1); + + AutoscalingTester tester = new AutoscalingTester(hosts); + NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(), + new OrchestratorMock(), + new MockHttpClient(tester.clock())); + Autoscaler autoscaler = new Autoscaler(new MockHostResourcesCalculator(), tester.nodeMetricsDb(), tester.nodeRepository()); + + ApplicationId application1 = tester.applicationId("test1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "test"); + Set<String> hostnames = tester.deploy(application1, cluster1, 2, 1, nodes) + .stream().map(HostSpec::hostname) + .collect(Collectors.toSet()); + // The metrics response (below) hardcodes these hostnames: + assertEquals(Set.of("node-1-of-host-1.yahoo.com", "node-1-of-host-10.yahoo.com"), hostnames); + + for (int i = 0; i < 1000; i++) { + tester.clock().advance(Duration.ofSeconds(10)); + tester.nodeMetricsDb().add(fetcher.fetchMetrics(application1)); + tester.clock().advance(Duration.ofSeconds(10)); + tester.nodeMetricsDb().gc(tester.clock()); + } + + var scaledResources = autoscaler.autoscale(application1, cluster1, tester.nodeRepository().getNodes(application1)); + assertTrue(scaledResources.isPresent()); + } + + private static class MockHttpClient implements NodeMetricsFetcher.HttpClient { + + private final ManualClock clock; + + public MockHttpClient(ManualClock clock) { + this.clock = clock; + } + + final String cannedResponse = + "{\n" + + " \"nodes\": [\n" + + " {\n" + + " \"hostname\": \"node-1-of-host-1.yahoo.com\",\n" + + " \"role\": \"role0\",\n" + + " \"node\": {\n" + + " \"timestamp\": [now],\n" + + " \"metrics\": [\n" + + " {\n" + + " \"values\": {\n" + + " \"cpu.util\": 16.2,\n" + + " \"mem.util\": 23.1,\n" + + " \"disk.util\": 82\n" + + " },\n" + + " \"dimensions\": {\n" + + " \"state\": \"active\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"hostname\": \"node-1-of-host-10.yahoo.com\",\n" + + " \"role\": \"role1\",\n" + + " \"node\": {\n" + + " \"timestamp\": [now],\n" + + " \"metrics\": [\n" + + " {\n" + + " \"values\": {\n" + + " \"cpu.util\": 20,\n" + + " \"mem.util\": 23.1,\n" + + " \"disk.util\": 40\n" + + " },\n" + + " \"dimensions\": {\n" + + " \"state\": \"active\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " ]\n" + + "}\n"; + + @Override + public String get(String url) { return cannedResponse.replace("[now]", String.valueOf(clock.millis())); } + + @Override + public void close() { } + + } + + private static class MockHostResourcesCalculator implements HostResourcesCalculator { + + @Override + public NodeResources realResourcesOf(Node node) { return node.flavor().resources(); } + + @Override + public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); } + + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index a4b174cdb29..015bf60113a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -86,12 +86,13 @@ class AutoscalingTester { deploy(application, cluster, resources.nodes(), resources.groups(), resources.advertisedResources()); } - public void deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { + public List<HostSpec> deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.fromCount(nodes, resources), groups); for (HostSpec host : hosts) makeReady(host.hostname()); provisioningTester.deployZoneApp(); provisioningTester.activate(application, hosts); + return hosts; } public void makeReady(String hostname) { @@ -137,7 +138,7 @@ class AutoscalingTester { db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), r.metricName(), clock().instant().toEpochMilli(), - effectiveValue))); + effectiveValue * 100))); // the metrics are in % } } } @@ -169,6 +170,8 @@ class AutoscalingTester { return provisioningTester.nodeRepository(); } + public NodeMetricsDb nodeMetricsDb() { return db; } + private static FlavorsConfig asConfig(NodeResources hostResources) { FlavorsConfig.Builder b = new FlavorsConfig.Builder(); b.flavor(asFlavorConfig("hostFlavor", hostResources)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java index 4376bfd38b0..4f039582125 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java @@ -41,7 +41,7 @@ public class NodeMetricsFetcherTest { httpClient.requestsReceived.get(0)); assertEquals(5, values.size()); assertEquals("metric value cpu.util: 16.2 at 1234 for host-1.yahoo.com", values.get(0).toString()); - assertEquals("metric value memory.util: 23.1 at 1234 for host-1.yahoo.com", values.get(1).toString()); + assertEquals("metric value mem.util: 23.1 at 1234 for host-1.yahoo.com", values.get(1).toString()); assertEquals("metric value disk.util: 82.0 at 1234 for host-1.yahoo.com", values.get(2).toString()); assertEquals("metric value cpu.util: 20.0 at 1200 for host-2.yahoo.com", values.get(3).toString()); assertEquals("metric value disk.util: 40.0 at 1200 for host-2.yahoo.com", values.get(4).toString()); @@ -54,7 +54,7 @@ public class NodeMetricsFetcherTest { httpClient.requestsReceived.get(1)); assertEquals(3, values.size()); assertEquals("metric value cpu.util: 10.0 at 1300 for host-3.yahoo.com", values.get(0).toString()); - assertEquals("metric value memory.util: 15.0 at 1300 for host-3.yahoo.com", values.get(1).toString()); + assertEquals("metric value mem.util: 15.0 at 1300 for host-3.yahoo.com", values.get(1).toString()); assertEquals("metric value disk.util: 20.0 at 1300 for host-3.yahoo.com", values.get(2).toString()); } } @@ -87,7 +87,7 @@ public class NodeMetricsFetcherTest { " {\n" + " \"values\": {\n" + " \"cpu.util\": 16.2,\n" + - " \"memory.util\": 23.1,\n" + + " \"mem.util\": 23.1,\n" + " \"disk.util\": 82\n" + " },\n" + " \"dimensions\": {\n" + @@ -118,7 +118,6 @@ public class NodeMetricsFetcherTest { " ]\n" + "}\n"; - final String cannedResponseForApplication2 = "{\n" + " \"nodes\": [\n" + @@ -131,7 +130,7 @@ public class NodeMetricsFetcherTest { " {\n" + " \"values\": {\n" + " \"cpu.util\": 10,\n" + - " \"memory.util\": 15,\n" + + " \"mem.util\": 15,\n" + " \"disk.util\": 20\n" + " },\n" + " \"dimensions\": {\n" + diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 9c2903c7aef..c4f6ef3b94e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -418,10 +418,13 @@ public class ProvisioningTester { activate(applicationId, Set.copyOf(list)); } + public ClusterSpec clusterSpec() { + return ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), + Version.fromString("6.42"), false, Optional.empty()); + } + public List<Node> deploy(ApplicationId application, Capacity capacity) { - ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), - Version.fromString("6.42"), false, Optional.empty()); - List<HostSpec> prepared = prepare(application, cluster, capacity, 1); + List<HostSpec> prepared = prepare(application, clusterSpec(), capacity, 1); activate(application, Set.copyOf(prepared)); return getNodes(application, Node.State.active).asList(); } |