Add mainainers to annotate deployments with cluster info and utilization

8 files changed, 343 insertions, 10 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 2ce2e480fc5..79994b6ad9c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -332,7 +332,16 @@ public class ApplicationController {
                     configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), 
                                                applicationPackage.zippedContent());
             preparedApplication.activate();
-            application = application.with(new Deployment(zone, revision, version, clock.instant()));
+
+            Deployment previousDeployment = application.deployments().get(zone);
+            Deployment newDeployment = previousDeployment;
+            if (previousDeployment == null) {
+                newDeployment = new Deployment(zone, revision, version, clock.instant(), new HashMap<>(), new HashMap<>());
+            } else {
+                newDeployment = new Deployment(zone, revision, version, clock.instant(), previousDeployment.clusterUtils(), previousDeployment.clusterInfo());
+            }
+
+            application = application.with(newDeployment);
             store(application, lock);
 
             return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
index 569e97e4f53..60f8bcfe3d5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
@@ -16,7 +16,7 @@ public class ClusterInfo {
     private final ClusterSpec.Type clusterType;
     private final List<String> hostnames;
 
-    ClusterInfo(Flavor flavor, ClusterSpec.Type clusterType, List<String> hostnames) {
+    public ClusterInfo(Flavor flavor, ClusterSpec.Type clusterType, List<String> hostnames) {
         this.flavor = flavor;
         this.clusterType = clusterType;
         this.hostnames = hostnames;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index f537206d0fb..915f3e81b90 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -21,10 +21,14 @@ public class Deployment {
     private final ApplicationRevision revision;
     private final Version version;
     private final Instant deployTime;
-    private final Map<Id, ClusterUtilization> clusterUtils = new HashMap<>();
-    private final Map<Id, ClusterInfo> clusterInfo = new HashMap<>();
+    private final Map<Id, ClusterUtilization> clusterUtils;
+    private final Map<Id, ClusterInfo> clusterInfo;
 
     public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) {
+        this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>());
+    }
+
+    public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo) {
         Objects.requireNonNull(zone, "zone cannot be null");
         Objects.requireNonNull(revision, "revision cannot be null");
         Objects.requireNonNull(version, "version cannot be null");
@@ -33,6 +37,8 @@ public class Deployment {
         this.revision = revision;
         this.version = version;
         this.deployTime = deployTime;
+        this.clusterUtils = clusterUtils;
+        this.clusterInfo = clusterInfo;
     }
 
     /** Returns the zone this was deployed to */
@@ -47,12 +53,20 @@ public class Deployment {
     /** Returns the time this was deployed */
     public Instant at() { return deployTime; }
 
-    public Map<Id, ClusterUtilization> getClusterUtilization() {
+    public  Map<Id, ClusterInfo> clusterInfo() {
+        return clusterInfo;
+    }
+
+    public  Map<Id, ClusterUtilization> clusterUtils() {
         return clusterUtils;
     }
 
-    public Map<Id, ClusterInfo> getClusterInfo() {
-        return clusterInfo;
+    public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) {
+        return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo);
+    }
+
+    public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) {
+        return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo);
     }
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
new file mode 100644
index 00000000000..af512db67f5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
@@ -0,0 +1,251 @@
+package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+
+import java.io.IOException;
+import java.net.URI;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Fetch info about hardware, hostnames and cluster specifications and update applications.
+ *
+ * @author smorgrav
+ */
+public class ClusterInfoMaintainer extends Maintainer {
+
+    private final Controller controller;
+
+    public ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) {
+        super(controller, duration, jobControl);
+        this.controller = controller;
+    }
+
+    private  Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeRepositoryJsonModel nodes, NodeFlavors flavors) {
+        Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>();
+
+        Map<String, List<NodeRepositoryJsonModel.Node>> clusters = nodes.nodes.stream()
+                .filter(node -> node.membership != null)
+                .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterid));
+
+        for (String id : clusters.keySet()) {
+            List<NodeRepositoryJsonModel.Node> clusterNodes = clusters.get(id);
+
+            //Assume they are all equal and use first node as a representatitve for the cluster
+            NodeRepositoryJsonModel.Node node = clusterNodes.get(0);
+
+            Optional<Flavor> flavorOpt = flavors.getFlavor(node.nodeFlavor);
+
+            List<String> hostnames = clusterNodes.stream().map(node1 -> { return node1.hostname; }).collect(Collectors.toList());
+
+            ClusterInfo inf = new ClusterInfo(flavorOpt.get(), ClusterSpec.Type.from(node.membership.clusterType), hostnames);
+
+            infoMap.put(new ClusterSpec.Id(id), inf);
+        }
+
+        return infoMap;
+    }
+
+    private static String clusterid(NodeRepositoryJsonModel.Node node) {
+        return node.membership.clusterId;
+
+    }
+
+    private NodeRepositoryJsonModel getApplicationNodes(ApplicationId appId, Zone zone) {
+        NodeRepositoryJsonModel nodesResponse = null;
+        ObjectMapper mapper = new ObjectMapper();
+        for (URI uri : controller.getConfigServerUris(null, null)) {
+            try {
+                nodesResponse = mapper.readValue(uri.toURL(), NodeRepositoryJsonModel.class);
+                break;
+            } catch (IOException ioe) {
+
+            }
+        }
+        return nodesResponse;
+    }
+
+    @Override
+    protected void maintain() {
+        for (Application application : controller().applications().asList()) {
+            for (Deployment deployment : application.deployments().values()) {
+                Zone zone = deployment.zone();
+                if (zone.nodeFlavors().isPresent()) {
+                    NodeFlavors flavors = zone.nodeFlavors().get();
+                    NodeRepositoryJsonModel appNodes = getApplicationNodes(application.id(), zone);
+
+                    Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(appNodes, flavors);
+                    application.with(deployment.withClusterInfo(clusterInfo));
+                }
+            }
+        }
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class NodeRepositoryJsonModel {
+
+        public final List<Node> nodes;
+
+        @JsonCreator
+        public NodeRepositoryJsonModel(@JsonProperty("nodes") List<Node> nodes) {
+            this.nodes = Collections.unmodifiableList(nodes);
+        }
+
+        @JsonIgnoreProperties(ignoreUnknown = true)
+        public static class Node {
+
+            public final String hostname;
+            public final String wantedDockerImage;
+            public final String currentDockerImage;
+            public final String nodeState;
+            public final String nodeType;
+            public final String nodeFlavor;
+            public final String nodeCanonicalFlavor;
+            public final String wantedVespaVersion;
+            public final String vespaVersion;
+            public final Owner owner;
+            public final Membership membership;
+            public final Long wantedRestartGeneration;
+            public final Long currentRestartGeneration;
+            public final Long wantedRebootGeneration;
+            public final Long currentRebootGeneration;
+            public final Double minCpuCores;
+            public final Double minMainMemoryAvailableGb;
+            public final Double minDiskAvailableGb;
+
+            @JsonCreator
+            public Node(@JsonProperty("id") String hostname,
+                        @JsonProperty("wantedDockerImage") String wantedDockerImage,
+                        @JsonProperty("currentDockerImage") String currentDockerImage,
+                        @JsonProperty("state") String nodeState,
+                        @JsonProperty("type") String nodeType,
+                        @JsonProperty("flavor") String nodeFlavor,
+                        @JsonProperty("canonicalFlavor") String nodeCanonicalFlavor,
+                        @JsonProperty("wantedVespaVersion") String wantedVespaVersion,
+                        @JsonProperty("vespaVersion") String vespaVersion,
+                        @JsonProperty("owner") Owner owner,
+                        @JsonProperty("membership") Membership membership,
+                        @JsonProperty("restartGeneration") Long wantedRestartGeneration,
+                        @JsonProperty("currentRestartGeneration") Long currentRestartGeneration,
+                        @JsonProperty("rebootGeneration") Long wantedRebootGeneration,
+                        @JsonProperty("currentRebootGeneration") Long currentRebootGeneration,
+                        @JsonProperty("minCpuCores") Double minCpuCores,
+                        @JsonProperty("minMainMemoryAvailableGb") Double minMainMemoryAvailableGb,
+                        @JsonProperty("minDiskAvailableGb") Double minDiskAvailableGb) {
+                this.hostname = hostname;
+                this.wantedDockerImage = wantedDockerImage;
+                this.currentDockerImage = currentDockerImage;
+                this.nodeState = nodeState;
+                this.nodeType = nodeType;
+                this.nodeFlavor = nodeFlavor;
+                this.nodeCanonicalFlavor = nodeCanonicalFlavor;
+                this.wantedVespaVersion = wantedVespaVersion;
+                this.vespaVersion = vespaVersion;
+                this.owner = owner;
+                this.membership = membership;
+                this.wantedRestartGeneration = wantedRestartGeneration;
+                this.currentRestartGeneration = currentRestartGeneration;
+                this.wantedRebootGeneration = wantedRebootGeneration;
+                this.currentRebootGeneration = currentRebootGeneration;
+                this.minCpuCores = minCpuCores;
+                this.minMainMemoryAvailableGb = minMainMemoryAvailableGb;
+                this.minDiskAvailableGb = minDiskAvailableGb;
+            }
+
+            public String toString() {
+                return "Node {"
+                        + " containerHostname = " + hostname
+                        + " wantedDockerImage = " + wantedDockerImage
+                        + " currentDockerImage = " + currentDockerImage
+                        + " nodeState = " + nodeState
+                        + " nodeType = " + nodeType
+                        + " nodeFlavor = " + nodeFlavor
+                        + " wantedVespaVersion = " + wantedVespaVersion
+                        + " vespaVersion = " + vespaVersion
+                        + " owner = " + owner
+                        + " membership = " + membership
+                        + " wantedRestartGeneration = " + wantedRestartGeneration
+                        + " currentRestartGeneration = " + currentRestartGeneration
+                        + " wantedRebootGeneration = " + wantedRebootGeneration
+                        + " currentRebootGeneration = " + currentRebootGeneration
+                        + " minCpuCores = " + minCpuCores
+                        + " minMainMemoryAvailableGb = " + minMainMemoryAvailableGb
+                        + " minDiskAvailableGb = " + minDiskAvailableGb
+                        + " }";
+            }
+
+
+            public static class Owner {
+                public final String tenant;
+                public final String application;
+                public final String instance;
+
+                public Owner(
+                        @JsonProperty("tenant") String tenant,
+                        @JsonProperty("application") String application,
+                        @JsonProperty("instance") String instance) {
+                    this.tenant = tenant;
+                    this.application = application;
+                    this.instance = instance;
+                }
+
+                public String toString() {
+                    return "Owner {" +
+                            " tenant = " + tenant +
+                            " application = " + application +
+                            " instance = " + instance +
+                            " }";
+                }
+            }
+
+            public static class Membership {
+                public final String clusterType;
+                public final String clusterId;
+                public final String group;
+                public final int index;
+                public final boolean retired;
+
+                public Membership(
+                        @JsonProperty("clustertype") String clusterType,
+                        @JsonProperty("clusterid") String clusterId,
+                        @JsonProperty("group") String group,
+                        @JsonProperty("index") int index,
+                        @JsonProperty("retired") boolean retired) {
+                    this.clusterType = clusterType;
+                    this.clusterId = clusterId;
+                    this.group = group;
+                    this.index = index;
+                    this.retired = retired;
+                }
+
+                @Override
+                public String toString() {
+                    return "Membership {" +
+                            " clusterType = " + clusterType +
+                            " clusterId = " + clusterId +
+                            " group = " + group +
+                            " index = " + index +
+                            " retired = " + retired +
+                            " }";
+                }
+            }
+        }
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
new file mode 100644
index 00000000000..0986b0abf7c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java
@@ -0,0 +1,53 @@
+package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
+import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Fetch utilization metrics and update applications with this data.
+ *
+ * @author smorgrav
+ */
+public class ClusterUtilizationMaintainer extends Maintainer {
+
+    Controller controller;
+
+    public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) {
+        super(controller, duration, jobControl);
+        this.controller = controller;
+    }
+
+    private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, Zone zone) {
+        List<MetricsService.ClusterCostMetrics> systemMetrics = controller.metricsService().getClusterCostMetrics(app, zone);
+
+        Map<ClusterSpec.Id, ClusterUtilization> utilizationMap = new HashMap<>();
+        for (MetricsService.ClusterCostMetrics clusterCostMetrics : systemMetrics) {
+            MetricsService.CostMetrics systemMetric = clusterCostMetrics.costMetrics();
+            ClusterUtilization utilization = new ClusterUtilization(systemMetric.memUtil() / 100, systemMetric.cpuUtil() / 100, systemMetric.diskUtil(), 0);
+            utilizationMap.put(new ClusterSpec.Id(clusterCostMetrics.clusterId()), utilization);
+        }
+
+        return utilizationMap;
+    }
+
+    @Override
+    protected void maintain() {
+        for (Application application : controller().applications().asList()) {
+            for (Deployment deployment : application.deployments().values()) {
+                Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone());
+                application.with(deployment.withClusterUtils(clusterUtilization));
+            }
+        }
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 7788821f04d..880abaaa6f9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -33,6 +33,8 @@ public class ControllerMaintenance extends AbstractComponent {
     private final Upgrader upgrader;
     private final DelayedDeployer delayedDeployer;
     private final BlockedChangeDeployer blockedChangeDeployer;
+    private final ClusterInfoMaintainer clusterInfoMaintainer;
+    private final ClusterUtilizationMaintainer clusterUtilizationMaintainer;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
     public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator,
@@ -50,6 +52,8 @@ public class ControllerMaintenance extends AbstractComponent {
         upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
         delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl);
         blockedChangeDeployer = new BlockedChangeDeployer(controller, maintenanceInterval, jobControl);
+        clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl);
+        clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl);
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -68,6 +72,8 @@ public class ControllerMaintenance extends AbstractComponent {
         upgrader.deconstruct();
         delayedDeployer.deconstruct();
         blockedChangeDeployer.deconstruct();
+        clusterUtilizationMaintainer.deconstruct();
+        clusterInfoMaintainer.deconstruct();
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 81b3cb635ef..76e0d3bf6db 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 
@@ -191,7 +192,8 @@ public class ApplicationSerializer {
         return new Deployment(zoneFromSlime(deploymentObject.field(zoneField)),
                               applicationRevisionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(),
                               Version.fromString(deploymentObject.field(versionField).asString()),
-                              Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()));
+                              Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()),
+                new HashMap<>(), new HashMap<>()); //TODO
     }
     
     private Zone zoneFromSlime(Inspector object) {
diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java
index 57506e44961..465d42a1f1d 100644
--- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java
+++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/verification/commons/noderepo/NodeRepoInfoRetriever.java
@@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 
 import java.io.IOException;
 import java.net.URL;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -27,7 +26,6 @@ public class NodeRepoInfoRetriever {
             try {
                 nodeRepoJsonModel = objectMapper.readValue(nodeInfoURL, NodeRepoJsonModel.class);
                 return nodeRepoJsonModel;
-
             } catch (IOException e) {
                 logger.log(Level.WARNING, "Failed to parse JSON from config server: " + nodeInfoURL.toString(), e);
             }