Convert percentages to ratio

9 files changed, 166 insertions, 18 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
index 93d4ad5bc47..4830d85f35d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -86,6 +86,7 @@ public class Autoscaler {
                                                                                   cluster);
         if (bestAllocation.isEmpty()) {
             log.fine("Autoscaling " + applicationId + " " + cluster + ": Could not find a better allocation");
+            System.out.println("Autoscaling " + applicationId + " " + cluster + ": Could not find a better allocation");
             return Optional.empty();
         }
 
@@ -94,6 +95,7 @@ public class Autoscaler {
             closeToIdeal(Resource.disk, diskLoad.get()) &&
             similarCost(bestAllocation.get().cost(), currentAllocation.cost())) {
             log.fine("Autoscaling " + applicationId + " " + cluster + ": Resources are almost ideal and price difference is small");
+            System.out.println("Autoscaling " + applicationId + " " + cluster + ": Resources are almost ideal and price difference is small");
             return Optional.empty(); // Avoid small, unnecessary changes
         }
         return bestAllocation;
@@ -132,10 +134,12 @@ public class Autoscaler {
         NodeResources nodeResources = nodeResourceLimits.enlargeToLegal(resources.nodeResources(), cluster.type());
         if (allowsHostSharing(nodeRepository.zone().cloud())) {
             // return the requested resources, or empty if they cannot fit on existing hosts
-            for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors())
+            for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) {
+                System.out.println("Host flavor: " + flavor.resources() + " wanted " + resources);
                 if (flavor.resources().satisfies(nodeResources))
                     return Optional.of(new AllocatableClusterResources(resources.with(nodeResources),
                                                                        nodeResources));
+            }
             return Optional.empty();
         }
         else {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
index a599606c314..10a78f8fdd7 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -47,12 +47,15 @@ public class MetricsResponse {
         long timestamp = node.field("timestamp").asLong();
         Map<String, Double> values = consumeMetrics(node.field("metrics"));
         for (Resource resource : Resource.values())
-            addMetricIfPresent(hostname, resource.metricName(), timestamp, values);
+            addMetricIfPresent(hostname, resource, timestamp, values);
     }
 
-    private void addMetricIfPresent(String hostname, String metricName, long timestamp, Map<String, Double> values) {
-        if (values.containsKey(metricName))
-            metricValues.add(new NodeMetrics.MetricValue(hostname, metricName, timestamp, values.get(metricName).floatValue()));
+    private void addMetricIfPresent(String hostname, Resource resource, long timestamp, Map<String, Double> values) {
+        if (values.containsKey(resource.metricName()))
+            metricValues.add(new NodeMetrics.MetricValue(hostname,
+                                                         resource.metricName(),
+                                                         timestamp,
+                                                         (float)resource.valueFromMetric(values.get(resource.metricName()))));
     }
 
     private void consumeServiceMetrics(String hostname, Inspector node) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
index 14a35e3efbc..44c6d5075c0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
@@ -164,6 +164,9 @@ public class NodeMetricsDb {
             this.value = value;
         }
 
+        @Override
+        public String toString() { return "measurement at " + timestamp + ": " + value; }
+
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
index 917ac6d3796..e0ecbcfbf3c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
@@ -38,6 +38,7 @@ public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics
     private final HttpClient httpClient;
 
     @Inject
+    @SuppressWarnings("unused")
     public NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator) {
         this(nodeRepository, orchestrator, new ApacheHttpClient());
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
index 1f8f0b2eefa..0eac14d61ca 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
@@ -10,22 +10,28 @@ import com.yahoo.config.provision.NodeResources;
  */
 public enum Resource {
 
+    /** Cpu utilization ratio */
     cpu {
         String metricName() { return "cpu.util"; }
         double idealAverageLoad() { return 0.2; }
         double valueFrom(NodeResources resources) { return resources.vcpu(); }
+        double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio
     },
 
+    /** Memory utilization ratio */
     memory {
         String metricName() { return "mem.util"; }
         double idealAverageLoad() { return 0.7; }
         double valueFrom(NodeResources resources) { return resources.memoryGb(); }
+        double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio
     },
 
+    /** Disk utilization ratio */
     disk {
         String metricName() { return "disk.util"; }
         double idealAverageLoad() { return 0.7; }
         double valueFrom(NodeResources resources) { return resources.diskGb(); }
+        double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio
     };
 
     abstract String metricName();
@@ -35,6 +41,8 @@ public enum Resource {
 
     abstract double valueFrom(NodeResources resources);
 
+    abstract double valueFromMetric(double metricValue);
+
     public static Resource fromMetric(String metricName) {
         for (Resource resource : values())
             if (resource.metricName().equals(metricName)) return resource;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
new file mode 100644
index 00000000000..781b48e9561
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
@@ -0,0 +1,124 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class AutoscalingIntegrationTest {
+
+    @Test
+    public void testComponentIntegration() {
+        NodeResources nodes = new NodeResources(1, 10, 100, 1);
+        NodeResources hosts = new NodeResources(3, 20, 200, 1);
+
+        AutoscalingTester tester = new AutoscalingTester(hosts);
+        NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(),
+                                                            new OrchestratorMock(),
+                                                            new MockHttpClient(tester.clock()));
+        Autoscaler autoscaler = new Autoscaler(new MockHostResourcesCalculator(), tester.nodeMetricsDb(), tester.nodeRepository());
+
+        ApplicationId application1 = tester.applicationId("test1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "test");
+        Set<String> hostnames = tester.deploy(application1, cluster1, 2, 1, nodes)
+                                      .stream().map(HostSpec::hostname)
+                                      .collect(Collectors.toSet());
+        // The metrics response (below) hardcodes these hostnames:
+        assertEquals(Set.of("node-1-of-host-1.yahoo.com", "node-1-of-host-10.yahoo.com"), hostnames);
+
+        for (int i = 0; i < 1000; i++) {
+            tester.clock().advance(Duration.ofSeconds(10));
+            tester.nodeMetricsDb().add(fetcher.fetchMetrics(application1));
+            tester.clock().advance(Duration.ofSeconds(10));
+            tester.nodeMetricsDb().gc(tester.clock());
+        }
+
+        var scaledResources = autoscaler.autoscale(application1, cluster1, tester.nodeRepository().getNodes(application1));
+        assertTrue(scaledResources.isPresent());
+    }
+
+    private static class MockHttpClient implements NodeMetricsFetcher.HttpClient {
+
+        private final ManualClock clock;
+
+        public MockHttpClient(ManualClock clock) {
+            this.clock = clock;
+        }
+
+        final String cannedResponse =
+                "{\n" +
+                "  \"nodes\": [\n" +
+                "    {\n" +
+                "      \"hostname\": \"node-1-of-host-1.yahoo.com\",\n" +
+                "      \"role\": \"role0\",\n" +
+                "      \"node\": {\n" +
+                "        \"timestamp\": [now],\n" +
+                "        \"metrics\": [\n" +
+                "          {\n" +
+                "            \"values\": {\n" +
+                "              \"cpu.util\": 16.2,\n" +
+                "              \"mem.util\": 23.1,\n" +
+                "              \"disk.util\": 82\n" +
+                "            },\n" +
+                "            \"dimensions\": {\n" +
+                "              \"state\": \"active\"\n" +
+                "            }\n" +
+                "          }\n" +
+                "        ]\n" +
+                "      }\n" +
+                "    },\n" +
+                "    {\n" +
+                "      \"hostname\": \"node-1-of-host-10.yahoo.com\",\n" +
+                "      \"role\": \"role1\",\n" +
+                "      \"node\": {\n" +
+                "        \"timestamp\": [now],\n" +
+                "        \"metrics\": [\n" +
+                "          {\n" +
+                "            \"values\": {\n" +
+                "              \"cpu.util\": 20,\n" +
+                "              \"mem.util\": 23.1,\n" +
+                "              \"disk.util\": 40\n" +
+                "            },\n" +
+                "            \"dimensions\": {\n" +
+                "              \"state\": \"active\"\n" +
+                "            }\n" +
+                "          }\n" +
+                "        ]\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}\n";
+
+        @Override
+        public String get(String url) { return cannedResponse.replace("[now]", String.valueOf(clock.millis())); }
+
+        @Override
+        public void close() { }
+
+    }
+
+    private static class MockHostResourcesCalculator implements HostResourcesCalculator {
+
+        @Override
+        public NodeResources realResourcesOf(Node node) { return node.flavor().resources(); }
+
+        @Override
+        public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); }
+
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
index a4b174cdb29..f6c9ebdf380 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -86,12 +86,13 @@ class AutoscalingTester {
         deploy(application, cluster, resources.nodes(), resources.groups(), resources.advertisedResources());
     }
 
-    public void deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) {
+    public List<HostSpec> deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) {
         List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.fromCount(nodes, resources), groups);
         for (HostSpec host : hosts)
             makeReady(host.hostname());
         provisioningTester.deployZoneApp();
         provisioningTester.activate(application, hosts);
+        return hosts;
     }
 
     public void makeReady(String hostname) {
@@ -169,6 +170,8 @@ class AutoscalingTester {
         return provisioningTester.nodeRepository();
     }
 
+    public NodeMetricsDb nodeMetricsDb() { return db; }
+
     private static FlavorsConfig asConfig(NodeResources hostResources) {
         FlavorsConfig.Builder b = new FlavorsConfig.Builder();
         b.flavor(asFlavorConfig("hostFlavor", hostResources));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java
index f48a87d0b93..0a83d84fc41 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java
@@ -40,11 +40,11 @@ public class NodeMetricsFetcherTest {
             assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=vespa-consumer-metrics",
                          httpClient.requestsReceived.get(0));
             assertEquals(5, values.size());
-            assertEquals("metric value cpu.util: 16.2 at 1234 for host-1.yahoo.com", values.get(0).toString());
-            assertEquals("metric value mem.util: 23.1 at 1234 for host-1.yahoo.com", values.get(1).toString());
-            assertEquals("metric value disk.util: 82.0 at 1234 for host-1.yahoo.com", values.get(2).toString());
-            assertEquals("metric value cpu.util: 20.0 at 1200 for host-2.yahoo.com", values.get(3).toString());
-            assertEquals("metric value disk.util: 40.0 at 1200 for host-2.yahoo.com", values.get(4).toString());
+            assertEquals("metric value cpu.util: 0.162 at 1234 for host-1.yahoo.com", values.get(0).toString());
+            assertEquals("metric value mem.util: 0.231 at 1234 for host-1.yahoo.com", values.get(1).toString());
+            assertEquals("metric value disk.util: 0.82 at 1234 for host-1.yahoo.com", values.get(2).toString());
+            assertEquals("metric value cpu.util: 0.2 at 1200 for host-2.yahoo.com", values.get(3).toString());
+            assertEquals("metric value disk.util: 0.4 at 1200 for host-2.yahoo.com", values.get(4).toString());
         }
 
         {
@@ -53,9 +53,9 @@ public class NodeMetricsFetcherTest {
             assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=vespa-consumer-metrics",
                          httpClient.requestsReceived.get(1));
             assertEquals(3, values.size());
-            assertEquals("metric value cpu.util: 10.0 at 1300 for host-3.yahoo.com", values.get(0).toString());
-            assertEquals("metric value mem.util: 15.0 at 1300 for host-3.yahoo.com", values.get(1).toString());
-            assertEquals("metric value disk.util: 20.0 at 1300 for host-3.yahoo.com", values.get(2).toString());
+            assertEquals("metric value cpu.util: 0.1 at 1300 for host-3.yahoo.com", values.get(0).toString());
+            assertEquals("metric value mem.util: 0.15 at 1300 for host-3.yahoo.com", values.get(1).toString());
+            assertEquals("metric value disk.util: 0.2 at 1300 for host-3.yahoo.com", values.get(2).toString());
         }
     }
 
@@ -118,7 +118,6 @@ public class NodeMetricsFetcherTest {
             "  ]\n" +
             "}\n";
 
-
     final String cannedResponseForApplication2 =
             "{\n" +
             "  \"nodes\": [\n" +
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 9c2903c7aef..c4f6ef3b94e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -418,10 +418,13 @@ public class ProvisioningTester {
         activate(applicationId, Set.copyOf(list));
     }
 
+    public ClusterSpec clusterSpec() {
+        return ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"),
+                                   Version.fromString("6.42"), false, Optional.empty());
+    }
+
     public List<Node> deploy(ApplicationId application, Capacity capacity) {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"),
-                                                  Version.fromString("6.42"), false, Optional.empty());
-        List<HostSpec> prepared = prepare(application, cluster, capacity, 1);
+        List<HostSpec> prepared = prepare(application, clusterSpec(), capacity, 1);
         activate(application, Set.copyOf(prepared));
         return getNodes(application, Node.State.active).asList();
     }