Compute application stats

1 files changed, 97 insertions, 3 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java
index e11a57f04df..42bd6a5ef1f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java
@@ -1,11 +1,20 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
+import com.yahoo.collections.Pair;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.autoscale.Load;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeTimeseries;
 
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
@@ -18,10 +27,12 @@ public class NodeRepoStats {
 
     private final Load load;
     private final Load activeLoad;
+    private final List<ApplicationStats> applicationStats;
 
-    private NodeRepoStats(Load load, Load activeLoad) {
+    private NodeRepoStats(Load load, Load activeLoad, List<ApplicationStats> applicationStats) {
         this.load = load;
         this.activeLoad = activeLoad;
+        this.applicationStats = List.copyOf(applicationStats);
     }
 
     /**
@@ -33,12 +44,22 @@ public class NodeRepoStats {
     /** Returns the current average utilization in this node repo over all active nodes. */
     public Load activeLoad() { return activeLoad; }
 
+    /** Returns stats on each application, sorted by decreasing unutilized allocation measured in dollars per hour. */
+    public List<ApplicationStats> applicationStats() { return applicationStats; }
+
     public static NodeRepoStats computeOver(NodeRepository nodeRepository) {
         NodeList allNodes = nodeRepository.nodes().list();
+        List<NodeTimeseries> allNodeTimeseries = nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of());
+
+        Pair<Load, Load> load = computeLoad(allNodes, allNodeTimeseries);
+        List<ApplicationStats> applicationStats = computeApplicationStats(allNodes, allNodeTimeseries);
+        return new NodeRepoStats(load.getFirst(), load.getSecond(), applicationStats);
+    }
 
+    private static Pair<Load, Load> computeLoad(NodeList allNodes, List<NodeTimeseries> allNodeTimeseries) {
         NodeResources totalActiveResources = new NodeResources(0, 0, 0, 0);
         double cpu = 0, memory = 0, disk = 0;
-        for (var nodeTimeseries : nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of())) {
+        for (var nodeTimeseries : allNodeTimeseries) {
             Optional<Node> node = allNodes.node(nodeTimeseries.hostname());
             if (node.isEmpty() || node.get().state() != Node.State.active) continue;
 
@@ -62,7 +83,47 @@ public class NodeRepoStats {
         Load activeLoad = new Load(divide(cpu, totalActiveResources.vcpu()),
                                    divide(memory, totalActiveResources.memoryGb()),
                                    divide(disk, totalActiveResources.diskGb()));
-        return new NodeRepoStats(load, activeLoad);
+        return new Pair<>(load, activeLoad);
+    }
+
+    private static List<ApplicationStats> computeApplicationStats(NodeList allNodes,
+                                                                  List<NodeTimeseries> allNodeTimeseries) {
+        List<ApplicationStats> applicationStats = new ArrayList<>();
+        Map<String, NodeMetricSnapshot> snapshotsByHost = byHost(allNodeTimeseries);
+        for (var applicationNodes : allNodes.state(Node.State.active)
+                                            .nodeType(NodeType.tenant)
+                                            .matching(node -> ! node.allocation().get().owner().instance().isTester())
+                                            .groupingBy(node -> node.allocation().get().owner()).entrySet()) {
+
+            NodeResources totalResources = new NodeResources(0, 0, 0, 0);
+            double utilizedCost = 0;
+            double totalCpu = 0, totalMemory = 0, totalDisk = 0;
+            for (var node : applicationNodes.getValue()) {
+                var snapshot = snapshotsByHost.get(node.hostname());
+                if (snapshot == null) continue;
+                double cpu = snapshot.cpu() * node.resources().vcpu();
+                double memory = snapshot.memory() * node.resources().memoryGb();
+                double disk = snapshot.disk() * node.resources().diskGb();
+                utilizedCost += new NodeResources(cpu, memory, disk, 0).cost();
+                totalCpu += cpu;
+                totalMemory += memory;
+                totalDisk += disk;
+                totalResources = totalResources.add(node.resources().justNumbers());
+            }
+            Load load = new Load(divide(totalCpu, totalResources.vcpu()),
+                                 divide(totalMemory, totalResources.memoryGb()),
+                                 divide(totalDisk, totalResources.diskGb()));
+            applicationStats.add(new ApplicationStats(applicationNodes.getKey(), load, totalResources.cost(), utilizedCost));
+        }
+        Collections.sort(applicationStats);
+        return applicationStats;
+    }
+
+    private static Map<String, NodeMetricSnapshot> byHost(List<NodeTimeseries> allNodeTimeseries) {
+        Map<String, NodeMetricSnapshot> snapshots = new HashMap<>();
+        for (var nodeTimeseries : allNodeTimeseries)
+            nodeTimeseries.last().ifPresent(last -> snapshots.put(nodeTimeseries.hostname(), last));
+        return snapshots;
     }
 
     private static double divide(double a, double b) {
@@ -70,4 +131,37 @@ public class NodeRepoStats {
         return a / b;
     }
 
+    public static class ApplicationStats implements Comparable<ApplicationStats> {
+
+        private final ApplicationId id;
+        private final Load load;
+        private final double cost;
+        private final double utilizedCost;
+
+        public ApplicationStats(ApplicationId id, Load load, double cost, double utilizedCost) {
+            this.id = id;
+            this.load = load;
+            this.cost = cost;
+            this.utilizedCost = utilizedCost;
+        }
+
+        public ApplicationId id() { return id; }
+        public Load load() { return load; }
+
+        /** The standard cost of this application */
+        public double cost() { return cost; }
+
+        /** The amount of that cost which is currently utilized */
+        public double utilizedCost() { return utilizedCost; }
+
+        /** Cost - utilizedCost */
+        public double unutilizedCost() { return cost - utilizedCost; }
+
+        @Override
+        public int compareTo(NodeRepoStats.ApplicationStats other) {
+            return -Double.compare(this.unutilizedCost(), other.unutilizedCost());
+        }
+
+    }
+
 }