Olaa/resource metering maintainer (#8945)

* Metering maintainer

* Added test. Misc fixes

* Fix component id

* cd only

* Added description/author

10 files changed, 167 insertions, 45 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index cc9e4020dab..da0eedbdd36 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer;
 import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig;
 import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
@@ -53,6 +54,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final ContactInformationMaintainer contactInformationMaintainer;
     private final CostReportMaintainer costReportMaintainer;
     private final RoutingPolicyMaintainer routingPolicyMaintainer;
+    private final ResourceMeterMaintainer resourceMeterMaintainer;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
     public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator,
@@ -61,6 +63,7 @@ public class ControllerMaintenance extends AbstractComponent {
                                  NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient,
                                  ContactRetriever contactRetriever,
                                  CostReportConsumer reportConsumer,
+                                 ResourceSnapshotConsumer resourceSnapshotConsumer,
                                  SelfHostedCostConfig selfHostedCostConfig) {
         Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
         this.jobControl = jobControl;
@@ -83,6 +86,7 @@ public class ControllerMaintenance extends AbstractComponent {
         contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever);
         costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig);
         routingPolicyMaintainer = new RoutingPolicyMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService, curator);
+        resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(5), jobControl, nodeRepositoryClient, Clock.systemUTC(), resourceSnapshotConsumer);
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -111,6 +115,7 @@ public class ControllerMaintenance extends AbstractComponent {
         contactInformationMaintainer.deconstruct();
         costReportMaintainer.deconstruct();
         routingPolicyMaintainer.deconstruct();
+        resourceMeterMaintainer.deconstruct();
     }
 
     /** Create one OS upgrader per cloud found in the zone registry of controller */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
index 2298c3c92fe..a76d472cc89 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
@@ -46,6 +46,6 @@ public class CostReportMaintainer extends Maintainer {
 
     @Override
     protected void maintain() {
-        consumer.Consume(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller(), clock, selfHostedCostConfig)));
+        consumer.Consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, selfHostedCostConfig));
     }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
new file mode 100644
index 00000000000..d9bd3e8131b
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -0,0 +1,92 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
+import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * Creates a ResourceSnapshot per application, which is then passed on to a ResourceSnapshotConsumer
+ * TODO: Write JSON blob of node repo somewhere
+ * @author olaa
+ */
+public class ResourceMeterMaintainer extends Maintainer {
+
+    private final Clock clock;
+    private final NodeRepositoryClientInterface nodeRepository;
+    private final ResourceSnapshotConsumer resourceSnapshotConsumer;
+
+    public ResourceMeterMaintainer(Controller controller,
+                                   Duration interval,
+                                   JobControl jobControl,
+                                   NodeRepositoryClientInterface nodeRepository,
+                                   Clock clock,
+                                   ResourceSnapshotConsumer resourceSnapshotConsumer) {
+        super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd));
+        this.clock = clock;
+        this.nodeRepository = nodeRepository;
+        this.resourceSnapshotConsumer = resourceSnapshotConsumer;
+    }
+
+    @Override
+    protected void maintain() {
+        List<NodeRepositoryNode> nodes = getNodes();
+        Map<ApplicationId, ResourceAllocation> resourceAllocationByApplication = getResourceAllocationByApplication(nodes);
+
+        // For now, we're only interested in resource allocation
+        Instant timeStamp = clock.instant();
+        Map<ApplicationId, ResourceSnapshot> resourceSnapshots = resourceAllocationByApplication.entrySet().stream()
+                .collect(Collectors.toMap(
+                        e -> e.getKey(),
+                        e -> new ResourceSnapshot(e.getValue(), timeStamp))
+                );
+
+
+        resourceSnapshotConsumer.consume(resourceSnapshots);
+    }
+
+    private List<NodeRepositoryNode> getNodes() {
+        return controller().zoneRegistry().zones()
+                .reachable().ids().stream()
+                .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream()))
+                .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa"))
+                .collect(Collectors.toList());
+    }
+
+    private Map<ApplicationId, ResourceAllocation> getResourceAllocationByApplication(List<NodeRepositoryNode> nodes) {
+        Map<ApplicationId, List<NodeRepositoryNode>> applicationNodes = new HashMap<>();
+
+        nodes.stream().forEach(node -> applicationNodes.computeIfAbsent(applicationIdFromNodeOwner(node.getOwner()), n -> new ArrayList<>()).add(node));
+
+        return applicationNodes.entrySet().stream()
+                .collect(
+                        Collectors.toMap(
+                                entry -> entry.getKey(),
+                                entry -> ResourceAllocation.from(entry.getValue())
+                        )
+                );
+    }
+
+    private ApplicationId applicationIdFromNodeOwner(NodeOwner owner) {
+        return ApplicationId.from(owner.getTenant(), owner.getApplication(), owner.getInstance());
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
index 6d599d32cc6..444153089da 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
@@ -36,7 +36,7 @@ public class CostApiHandler extends LoggingRequestHandler {
         Path path = new Path(request.getUri().getPath());
 
         if (path.matches("/cost/v1/csv")) {
-            return new StringResponse(CostCalculator.toCsv(CostCalculator.calculateCost(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig)));
+            return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), selfHostedCostConfig));
         }
 
         return ErrorResponse.notFoundError("Nothing at " + path);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
index 88fe28a3613..fc30ecc97bb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
@@ -3,16 +3,20 @@ package com.yahoo.vespa.hosted.controller.restapi.cost;
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
 import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 
 import java.time.Clock;
 import java.time.LocalDate;
+import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -24,7 +28,7 @@ public class CostCalculator {
 
     private static final double SELF_HOSTED_DISCOUNT = .5;
 
-    public static Map<Property, ResourceAllocation> calculateCost(NodeRepositoryClientInterface nodeRepository,
+    public static String resourceShareByPropertyToCsv(NodeRepositoryClientInterface nodeRepository,
                                                                   Controller controller,
                                                                   Clock clock,
                                                                   SelfHostedCostConfig selfHostedCostConfig) {
@@ -50,7 +54,7 @@ public class CostCalculator {
             return selfHostedNode;
         }).forEach(nodes::add);
 
-        ResourceAllocation total = ResourceAllocation.from(date, nodes, null);
+        ResourceAllocation totalResourceAllocation = ResourceAllocation.from(nodes);
 
         Map<String, Property> propertyByTenantName = controller.tenants().asList().stream()
                 .filter(AthenzTenant.class::isInstance)
@@ -63,58 +67,29 @@ public class CostCalculator {
                 .map(SelfHostedCostConfig.Properties::name)
                 .forEach(name -> propertyByTenantName.put(name, new Property(name)));
 
-        return nodes.stream()
+        Map<Property, ResourceAllocation> resourceShareByProperty = nodes.stream()
                 .filter(node -> propertyByTenantName.containsKey(node.getOwner().tenant))
                 .collect(Collectors.groupingBy(
                         node -> propertyByTenantName.get(node.getOwner().tenant),
                         Collectors.collectingAndThen(
                                 Collectors.toList(),
-                                (tenantNodes) -> ResourceAllocation.from(date, tenantNodes, total)
+                                (tenantNodes) -> ResourceAllocation.from(tenantNodes)
                         )
                 ));
-    }
 
-    static class ResourceAllocation {
-        final double cpuCores;
-        final double memoryGb;
-        final double diskGb;
-        final String date;
-        final ResourceAllocation total;
-
-        private ResourceAllocation(String date, double cpuCores, double memoryGb, double diskGb, ResourceAllocation total) {
-            this.date = date;
-            this.cpuCores = cpuCores;
-            this.memoryGb = memoryGb;
-            this.diskGb = diskGb;
-            this.total = total;
-        }
-
-        private static ResourceAllocation from(String date, List<NodeRepositoryNode> nodes, ResourceAllocation total) {
-            return new ResourceAllocation(
-                    date,
-                    nodes.stream().mapToDouble(NodeRepositoryNode::getMinCpuCores).sum(),
-                    nodes.stream().mapToDouble(NodeRepositoryNode::getMinMainMemoryAvailableGb).sum(),
-                    nodes.stream().mapToDouble(NodeRepositoryNode::getMinDiskAvailableGb).sum(),
-                    total
-            );
-        }
-
-        private double usageFraction() {
-            return (cpuCores / total.cpuCores + memoryGb / total.memoryGb + diskGb / total.diskGb) / 3;
-        }
+        return toCsv(resourceShareByProperty, date, totalResourceAllocation);
     }
 
-    public static String toCsv(Map<Property, ResourceAllocation> resourceShareByProperty) {
+    private static String toCsv(Map<Property, ResourceAllocation> resourceShareByProperty, String date, ResourceAllocation totalResourceAllocation) {
         String header = "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n";
         String entries = resourceShareByProperty.entrySet().stream()
-                .sorted((Comparator.comparingDouble(entry -> entry.getValue().usageFraction())))
+                .sorted((Comparator.comparingDouble(entry -> entry.getValue().usageFraction(totalResourceAllocation))))
                 .map(propertyEntry -> {
                     ResourceAllocation r = propertyEntry.getValue();
-                    return Stream.of(r.date, propertyEntry.getKey(), r.cpuCores, r.memoryGb, r.diskGb, r.usageFraction())
+                    return Stream.of(date, propertyEntry.getKey(), r.getCpuCores(), r.getMemoryGb(), r.getDiskGb(), r.usageFraction(totalResourceAllocation))
                             .map(Object::toString).collect(Collectors.joining(","));
                 })
                 .collect(Collectors.joining("\n"));
         return header + entries;
     }
-
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java
index daddc46589d..ccd09cb9261 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java
@@ -56,8 +56,8 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface {
         node.setMinDiskAvailableGb(500d);
         node.setMinMainMemoryAvailableGb(24d);
         NodeOwner owner = new NodeOwner();
-        owner.tenant = "lsbe";
-        owner.application = "local-search";
+        owner.tenant = "tenant1";
+        owner.application = "app1";
         owner.instance = "default";
         node.setOwner(owner);
         NodeMembership membership = new NodeMembership();
@@ -76,8 +76,8 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface {
         node.setMinDiskAvailableGb(500d);
         node.setMinMainMemoryAvailableGb(24d);
         NodeOwner owner = new NodeOwner();
-        owner.tenant = "mediasearch";
-        owner.application = "imagesearch";
+        owner.tenant = "tenant2";
+        owner.application = "app2";
         owner.instance = "default";
         node.setOwner(owner);
         NodeMembership membership = new NodeMembership();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
index 01f3f55c679..890af974a0e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java
@@ -34,8 +34,8 @@ public class CostReportMaintainerTest {
                 .build();
 
 
-        tester.createTenant("lsbe", "local-search", 1L);
-        tester.createTenant("mediasearch", "msbe", 2L);
+        tester.createTenant("tenant1", "app1", 1L);
+        tester.createTenant("tenant2", "app2", 2L);
         CostReportMaintainer maintainer = new CostReportMaintainer(
                 tester.controller(),
                 Duration.ofDays(1),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
new file mode 100644
index 00000000000..14c75e791a5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
@@ -0,0 +1,46 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
+import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer;
+import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author olaa
+ */
+public class ResourceMeterMaintainerTest {
+
+    private final double DELTA = Double.MIN_VALUE;
+    NodeRepositoryClientMock nodeRepository = new NodeRepositoryClientMock();
+    MockResourceSnapshotConsumer snapshotConsumer = new MockResourceSnapshotConsumer();
+
+    @Test
+    public void testMaintainer() {
+        ControllerTester tester = new ControllerTester();
+        ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), snapshotConsumer);
+        resourceMeterMaintainer.maintain();
+        Map<ApplicationId, ResourceSnapshot> consumedResources = snapshotConsumer.consumedResources();
+
+        assertEquals(2, consumedResources.size());
+
+        ResourceSnapshot app1 = consumedResources.get(ApplicationId.from("tenant1", "app1", "default"));
+        ResourceSnapshot app2 = consumedResources.get(ApplicationId.from("tenant2", "app2", "default"));
+
+        assertEquals(96, app1.getResourceAllocation().getCpuCores(), DELTA);
+        assertEquals(96, app1.getResourceAllocation().getMemoryGb(), DELTA);
+        assertEquals(2000, app1.getResourceAllocation().getDiskGb(), DELTA);
+
+        assertEquals(160, app2.getResourceAllocation().getCpuCores(), DELTA);
+        assertEquals(96, app2.getResourceAllocation().getMemoryGb(), DELTA);
+        assertEquals(2000, app2.getResourceAllocation().getDiskGb(), DELTA);
+
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index 71b65770b1e..331a6ba9ac8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -75,6 +75,7 @@ public class ControllerContainerTest {
             "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler'/>\n" +
+            "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerMock'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock'/>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index dd64d480453..0b5d3912214 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -46,6 +46,9 @@
       "name": "ReadyJobsTrigger"
     },
     {
+      "name": "ResourceMeterMaintainer"
+    },
+    {
       "name": "RoutingPolicyMaintainer"
     },
     {