Controller side metric handling

11 files changed, 150 insertions, 9 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ClusterMetrics.java
new file mode 100644
index 00000000000..dd301c73b8d
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ClusterMetrics.java
@@ -0,0 +1,39 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author olaa
+ */
+public class ClusterMetrics {
+
+    private final String clusterId;
+    private final ClusterType clusterType;
+    private final Map<String, Double> metrics;
+
+    public ClusterMetrics(String clusterId, ClusterType clusterType) {
+        this.clusterId = clusterId;
+        this.clusterType = clusterType;
+        this.metrics = new HashMap<>();
+    }
+
+    public String getClusterId() {
+        return clusterId;
+    }
+
+    public ClusterType getClusterType() {
+        return clusterType;
+    }
+
+    public Map<String, Double> getMetrics() {
+        return metrics;
+    }
+
+    public void addMetric(String name, double value) {
+        metrics.put(name, value);
+    }
+
+    public enum ClusterType {content, container};
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ContainerClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ContainerClusterMetrics.java
new file mode 100644
index 00000000000..9bb7213f8f9
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ContainerClusterMetrics.java
@@ -0,0 +1,22 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics;
+
+/**
+ * @author olaa
+ */
+public class ContainerClusterMetrics {
+
+    private final double queriesPerSecond;
+    private final double writesPerSecond;
+    private final double queryLatencyMillis;
+    private final double writeLatencyMills;
+
+    public ContainerClusterMetrics(String clusterId, double queriesPerSecond, double writesPerSecond, double queryLatencyMillis, double writeLatencyMills) {
+        this.queriesPerSecond = queriesPerSecond;
+        this.writesPerSecond = writesPerSecond;
+        this.queryLatencyMillis = queryLatencyMillis;
+        this.writeLatencyMills = writeLatencyMills;
+    }
+
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ContentClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ContentClusterMetrics.java
new file mode 100644
index 00000000000..0ccc19e0172
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/metrics/ContentClusterMetrics.java
@@ -0,0 +1,16 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics;
+
+import com.yahoo.config.provision.ClusterSpec;
+
+/**
+ * @author olaa
+ */
+public class ContentClusterMetrics{
+
+    private final double documentCount;
+
+    public ContentClusterMetrics(String clusterId, double documentCount) {
+        this.documentCount = documentCount;
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index bcad82a23c1..6636b1ea73f 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 
 import java.io.IOException;
@@ -43,6 +45,8 @@ public interface ConfigServer {
 
     InputStream getLogs(DeploymentId deployment, Map<String, String> queryParameters);
 
+    Map<ApplicationId, List<ClusterMetrics>> getMetrics(ZoneId zoneId);
+
     List<String> getContentClusters(DeploymentId deployment);
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 943ede5197b..cc76b953572 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -156,7 +156,8 @@ public class LockedApplication {
                                                   previousDeployment.clusterUtils(),
                                                   previousDeployment.clusterInfo(),
                                                   previousDeployment.metrics().with(warnings),
-                                                  previousDeployment.activity());
+                                                  previousDeployment.activity(),
+                                                  previousDeployment.clusterMetrics());
         return with(newDeployment);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index b9be68f6de1..99da219c204 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -4,12 +4,14 @@ package com.yahoo.vespa.hosted.controller.application;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ClusterSpec.Id;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.config.provision.zone.ZoneId;
 
 import java.time.Instant;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -29,16 +31,18 @@ public class Deployment {
     private final Map<Id, ClusterInfo> clusterInfo;
     private final DeploymentMetrics metrics;
     private final DeploymentActivity activity;
+    private final List<ClusterMetrics> clusterMetrics;
 
     public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) {
         this(zone, applicationVersion, version, deployTime, Collections.emptyMap(), Collections.emptyMap(),
-             DeploymentMetrics.none, DeploymentActivity.none);
+             DeploymentMetrics.none, DeploymentActivity.none, Collections.emptyList());
     }
 
     public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime,
                       Map<Id, ClusterUtilization> clusterUtilization, Map<Id, ClusterInfo> clusterInfo,
                       DeploymentMetrics metrics,
-                      DeploymentActivity activity) {
+                      DeploymentActivity activity,
+                      List<ClusterMetrics> clusterMetrics) {
         this.zone = Objects.requireNonNull(zone, "zone cannot be null");
         this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null");
         this.version = Objects.requireNonNull(version, "version cannot be null");
@@ -47,6 +51,7 @@ public class Deployment {
         this.clusterInfo = ImmutableMap.copyOf(Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"));
         this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null");
         this.activity = Objects.requireNonNull(activity, "activity cannot be null");
+        this.clusterMetrics = Objects.requireNonNull(clusterMetrics, "cluster metrics cannot be null");
     }
 
     /** Returns the zone this was deployed to */
@@ -79,24 +84,28 @@ public class Deployment {
         return clusterUtilization;
     }
 
+    public List<ClusterMetrics> clusterMetrics() {
+        return clusterMetrics;
+    }
+
     public Deployment recordActivityAt(Instant instant) {
         return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics,
-                              activity.recordAt(instant, metrics));
+                              activity.recordAt(instant, metrics), clusterMetrics);
     }
 
     public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) {
         return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics,
-                              activity);
+                              activity, clusterMetrics);
     }
 
     public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) {
         return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, newClusterInfo, metrics,
-                              activity);
+                              activity, clusterMetrics);
     }
 
     public Deployment withMetrics(DeploymentMetrics metrics) {
         return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics,
-                              activity);
+                              activity, clusterMetrics);
     }
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java
index 7a50184e7a4..bda9683cbf1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application;
 
+import com.yahoo.config.provision.ClusterSpec;
+
 import java.time.Instant;
 import java.util.Map;
 import java.util.Objects;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 0cf89d798a7..f8dddff7850 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -1,10 +1,12 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
@@ -97,6 +99,7 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
         }
+        getClusterMetrics();
     }
 
     /** Get global rotation status for application */
@@ -120,4 +123,15 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         }
     }
 
+    private void getClusterMetrics() {
+        controller().zoneRegistry().zones()
+                .reachable().ids()
+                .stream().forEach(zoneId -> {
+                    Map<ApplicationId, List<ClusterMetrics>> allMetrics = controller().configServer().getMetrics(zoneId);
+
+                }
+        );
+    }
+
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 3c2cbade606..01187e0d69e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -14,6 +14,7 @@ import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
@@ -91,6 +92,7 @@ public class ApplicationSerializer {
     private final String lastWrittenField = "lastWritten";
     private final String lastQueriesPerSecondField = "lastQueriesPerSecond";
     private final String lastWritesPerSecondField = "lastWritesPerSecond";
+    private final String clusterMetricsField = "clusterMetrics";
 
     // DeploymentJobs fields
     private final String projectIdField = "projectId";
@@ -184,6 +186,7 @@ public class ApplicationSerializer {
         deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli()));
         deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value));
         deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value));
+        clusterMetricsToSlime(object.setObject(clusterMetricsField), deployment.clusterMetrics());
     }
 
     private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) {
@@ -310,6 +313,29 @@ public class ApplicationSerializer {
         });
     }
 
+    private void clusterMetricsToSlime(Cursor parent, List<ClusterMetrics> clusterMetricsList) {
+        Cursor metricsArray = parent.setArray("clusterMetrics");
+        for (ClusterMetrics clusterMetrics : clusterMetricsList) {
+            Cursor cluster = metricsArray.addObject();
+            cluster.setString("clusterId", clusterMetrics.getClusterId());
+            cluster.setString("clusterType", clusterMetrics.getClusterType().name());
+            Cursor metrics = cluster.setObject("metrics");
+            clusterMetrics.getMetrics().entrySet().stream().forEach(entry -> metrics.setDouble(entry.getKey(), entry.getValue()));
+        }
+    }
+
+    private List<ClusterMetrics> clusterMetricsFromSlime(Inspector inspector) {
+        List<ClusterMetrics> clusterMetricsList = new ArrayList<>();
+        inspector.traverse((ArrayTraverser) (i, cluster) -> {
+            String clusterId = cluster.field("clusterId").asString();
+            ClusterMetrics.ClusterType clusterType = ClusterMetrics.ClusterType.valueOf(cluster.field("clusterType").asString());
+            ClusterMetrics clusterMetrics = new ClusterMetrics(clusterId, clusterType);
+            cluster.field("metrics").traverse((ObjectTraverser) (name, value) -> clusterMetrics.addMetric(name, value.asDouble()));
+            clusterMetricsList.add(clusterMetrics);
+        });
+        return clusterMetricsList;
+    }
+
     // ------------------ Deserialization
 
     public Application fromSlime(Slime slime) {
@@ -354,7 +380,8 @@ public class ApplicationSerializer {
                               DeploymentActivity.create(optionalInstant(deploymentObject.field(lastQueriedField)),
                                                         optionalInstant(deploymentObject.field(lastWrittenField)),
                                                         optionalDouble(deploymentObject.field(lastQueriesPerSecondField)),
-                                                        optionalDouble(deploymentObject.field(lastWritesPerSecondField))));
+                                                        optionalDouble(deploymentObject.field(lastWritesPerSecondField))),
+                              clusterMetricsFromSlime(deploymentObject.field(clusterMetricsField)));
     }
 
     private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index e201258c701..9e016b847dd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.metrics.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
@@ -340,6 +341,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return applicationView;
     }
 
+    @Override
+    public Map<ApplicationId, List<ClusterMetrics>> getMetrics(ZoneId zoneId) {
+        return null;
+    }
+
     // Returns a canned example response
     @Override
     public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index d7c0ac9fe9e..674227a500e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -35,6 +35,7 @@ import java.nio.file.Paths;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -81,7 +82,7 @@ public class ApplicationSerializerTest {
                                                              Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)),
                                                              Map.of(DeploymentMetrics.Warning.all, 3)),
                                        DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt),
-                                                                 OptionalDouble.of(200), OptionalDouble.of(10))));
+                                                                 OptionalDouble.of(200), OptionalDouble.of(10)), Collections.emptyList()));
 
         OptionalLong projectId = OptionalLong.of(123L);
         List<JobStatus> statusList = new ArrayList<>();