Initial structure and functions (#13789)

* Initial structure and functions

* Finsihed Cluster. Working on MetricsAggregator

* WIP. Nearly fixed ClusterMetrics

* Debugging getMetrics issues

* Renaming and attempting to fix configServer issues

* Added some necessary functions. Finishing ConfigServer

* Finishing renaming and starting getting ready for impl

* Fixed bindings

* Finished pipeline. Added Metrics type

* Implemented necessary methods to make it pass testing, no unit tests yet

* Renamed versioning from metrics

* Extracted reused functions to new helper class

* Changed format and response structure of ProtonMetrics

* Removed list of metrics name

* fixed aggregation for proton metrics

* updated aggregator to use cluster format and more aligned with metric format

* added unit tests and resources

* Added correct return object

* fixed according to review. added second host. not tested

* removes superfluous roles

* Fixed ConfigServer to align with previous impl

* Updated test to use 2 different hosts and clusters

* Added processing to build proper JsonResponse for proton metrics

* changed proton metrics format to be consistent

* Moved response construction to api handler

* Removed unused import. Removed superfluos name

* Updated ConfigServerMock to match new ConfigServer

* developing metrics test

* added working unit test

* updated metrics path to ignore instance

5 files changed, 96 insertions, 3 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java
index 567cb664788..266af5e35fe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java
@@ -31,7 +31,7 @@ public class ConfigServerMetrics {
 
     public DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone) {
         var deploymentId = new DeploymentId(application, zone);
-        var metrics = configServer.getMetrics(deploymentId);
+        var metrics = configServer.getDeploymentMetrics(deploymentId);
 
         // The field names here come from the MetricsResponse class.
         return new DeploymentMetrics(
@@ -61,5 +61,4 @@ public class ConfigServerMetrics {
 
         return weightedLatency / rateSum;
     }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 427d1d66ee8..20aa96e890d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -18,6 +18,7 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.container.handler.metrics.JsonResponse;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
@@ -40,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.ActivateResult;
 import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo;
@@ -118,6 +120,9 @@ import java.util.Scanner;
 import java.util.StringJoiner;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
 
 import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
 import static com.yahoo.jdisc.Response.Status.CONFLICT;
@@ -228,6 +233,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/clusters")) return clusters(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/metrics")) return metrics(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), Optional.ofNullable(request.getProperty("endpointId")));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -647,6 +653,31 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         };
     }
 
+    private HttpResponse metrics(String tenantName, String applicationName, String instanceName, String environment, String region) {
+        ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName);
+        ZoneId zone = ZoneId.from(environment, region);
+        DeploymentId deployment = new DeploymentId(application, zone);
+        List<ProtonMetrics> protonMetrics = controller.serviceRegistry().configServer().getProtonMetrics(deployment);
+        return buildResponseFromProtonMetrics(protonMetrics);
+    }
+
+    private JsonResponse buildResponseFromProtonMetrics(List<ProtonMetrics> protonMetrics) {
+        try {
+            var jsonObject = new JSONObject();
+            var jsonArray = new JSONArray();
+            for (ProtonMetrics metrics : protonMetrics) {
+                jsonArray.put(metrics.toJson());
+            }
+            jsonObject.put("metrics", jsonArray);
+            return new JsonResponse(200, jsonObject.toString());
+        } catch (JSONException e) {
+            log.severe("Unable to build JsonResponse with Proton data");
+            return new JsonResponse(500, "");
+        }
+    }
+
+
+
     private HttpResponse trigger(ApplicationId id, JobType type, HttpRequest request) {
         Inspector requestObject = toSlime(request.getData()).get();
         boolean requireTests = ! requestObject.field("skipTests").asBool();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 251a5ce9acb..ec513e770fa 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.flags.json.FlagData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
@@ -33,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceCon
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.vespa.serviceview.bindings.ClusterView;
@@ -57,6 +59,7 @@ import java.util.UUID;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import org.json.JSONObject;
 
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
 import static com.yahoo.config.provision.NodeResources.StorageType.remote;
@@ -81,6 +84,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private final Map<DeploymentId, List<Log>> warnings = new HashMap<>();
     private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>();
     private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>();
+    private List<ProtonMetrics> protonMetrics;
 
     private Version lastPrepareVersion = null;
     private RuntimeException prepareException = null;
@@ -261,6 +265,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         this.clusterMetrics.put(deployment, clusterMetrics);
     }
 
+    public void setProtonMetrics(List<ProtonMetrics> protonMetrics) {
+        this.protonMetrics = protonMetrics;
+    }
+
     public void deferLoadBalancerProvisioningIn(Set<Environment> environments) {
         deferLoadBalancerProvisioning.addAll(environments);
     }
@@ -444,10 +452,15 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public List<ClusterMetrics> getMetrics(DeploymentId deployment) {
+    public List<ClusterMetrics> getDeploymentMetrics(DeploymentId deployment) {
         return Collections.unmodifiableList(clusterMetrics.getOrDefault(deployment, List.of()));
     }
 
+    @Override
+    public List<ProtonMetrics> getProtonMetrics(DeploymentId deployment) {
+        return this.protonMetrics;
+    }
+
     // Returns a canned example response
     @Override
     public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 10682218353..57397cfd856 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.RoutingController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
@@ -494,6 +495,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
                         .userIdentity(USER_ID),
                 "INFO - All good");
 
+        updateMetrics();
+
+        // GET metrics
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-central-1/instance/default/metrics", GET)
+                        .userIdentity(USER_ID),
+                                new File("proton-metrics.json"));
+
         // DELETE (cancel) ongoing change
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE)
                                       .userIdentity(HOSTED_VESPA_OPERATOR),
@@ -1584,6 +1592,25 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                                                                   "queue", Optional.empty()));
     }
 
+    private void updateMetrics() {
+        tester.serviceRegistry().configServerMock().setProtonMetrics(List.of(
+                (new ProtonMetrics("content/doc/"))
+                        .addMetric(ProtonMetrics.DOCUMENTS_ACTIVE_COUNT, 11430)
+                        .addMetric(ProtonMetrics.DOCUMENTS_READY_COUNT, 11430)
+                        .addMetric(ProtonMetrics.DOCUMENTS_TOTAL_COUNT, 11430)
+                        .addMetric(ProtonMetrics.DOCUMENT_DISK_USAGE, 44021)
+                        .addMetric(ProtonMetrics.RESOURCE_DISK_USAGE_AVERAGE, 0.0168421)
+                        .addMetric(ProtonMetrics.RESOURCE_MEMORY_USAGE_AVERAGE, 0.103482),
+                (new ProtonMetrics("content/music/"))
+                        .addMetric(ProtonMetrics.DOCUMENTS_ACTIVE_COUNT, 32210)
+                        .addMetric(ProtonMetrics.DOCUMENTS_READY_COUNT, 32000)
+                        .addMetric(ProtonMetrics.DOCUMENTS_TOTAL_COUNT, 32210)
+                        .addMetric(ProtonMetrics.DOCUMENT_DISK_USAGE, 90113)
+                        .addMetric(ProtonMetrics.RESOURCE_DISK_USAGE_AVERAGE, 0.23912)
+                        .addMetric(ProtonMetrics.RESOURCE_MEMORY_USAGE_AVERAGE, 0.00912)
+        ));
+    }
+
     private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) {
         var changedAt = tester.controller().clock().instant();
         var westPolicies = tester.controller().routing().policies().get(deployment);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json
new file mode 100644
index 00000000000..a7e5b3918d8
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json
@@ -0,0 +1,23 @@
+{
+  "metrics": [{
+    "clusterId": "content/doc/",
+    "metrics": {
+      "resourceMemoryUsageAverage": 0.103482,
+      "documentsReadyCount": 11430,
+      "documentDiskUsage": 44021,
+      "resourceDiskUsageAverage": 0.0168421,
+      "documentsTotalCount": 11430,
+      "documentsActiveCount": 11430
+    }
+  }, {
+    "clusterId": "content/music/",
+    "metrics": {
+      "resourceMemoryUsageAverage": 0.00912,
+      "documentsReadyCount": 32000,
+      "documentDiskUsage": 90113,
+      "resourceDiskUsageAverage": 0.23912,
+      "documentsTotalCount": 32210,
+      "documentsActiveCount": 32210
+    }
+  }]
+}
\ No newline at end of file