Cfg server retrieves metrics from all applications in zone

10 files changed, 382 insertions, 4 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index b4414562170..b2c2b85d358 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -43,6 +43,8 @@ import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream;
 import com.yahoo.vespa.config.server.http.LogRetriever;
 import com.yahoo.vespa.config.server.http.SimpleHttpFetcher;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
+import com.yahoo.vespa.config.server.metrics.Metrics;
+import com.yahoo.vespa.config.server.metrics.MetricsAggregator;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.LocalSession;
 import com.yahoo.vespa.config.server.session.LocalSessionRepo;
@@ -67,10 +69,13 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
@@ -80,6 +85,7 @@ import java.util.stream.Collectors;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
+import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
 import static java.nio.file.Files.readAttributes;
 
 /**
@@ -579,6 +585,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return tenantRepository.getTenant(tenantName).getApplicationRepo().activeApplications();
     }
 
+    // ---------------- Metrics ------------------------------------------------------------------------
+
+    public HttpResponse getMetrics() {
+        MetricsAggregator metricsAggregator = new MetricsAggregator();
+        Map<ApplicationId, Map<String, List<URI>>> applicationHosts = getHostsPerApplication();
+        return metricsAggregator.aggregateAllMetrics(applicationHosts);
+    }
+
     // ---------------- Misc operations ----------------------------------------------------------------
 
     public Tenant verifyTenantAndApplication(ApplicationId applicationId) {
@@ -717,18 +731,48 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                         .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver")))
                 .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find HostInfo for LogServer"));
 
-        ServiceInfo containerServiceInfo = logServerHostInfo.getServices().stream()
-                .filter(service -> List.of(LOGSERVER_CONTAINER.serviceName, CONTAINER.serviceName).contains(service.getServiceType()))
+        ServiceInfo serviceInfo = logServerHostInfo.getServices().stream().filter(service -> List.of(LOGSERVER_CONTAINER.serviceName, CONTAINER.serviceName).contains(service.getServiceType()))
                 .findFirst().orElseThrow(() -> new IllegalArgumentException("No container running on logserver host"));
+        int port = servicePort(serviceInfo);
+        return "http://" + logServerHostInfo.getHostname() + ":" + port + "/logs";
+    }
 
-        int port = containerServiceInfo.getPorts().stream()
+    private int servicePort(ServiceInfo serviceInfo) {
+        int port = serviceInfo.getPorts().stream()
                 .filter(portInfo -> portInfo.getTags().stream().anyMatch(tag -> tag.equalsIgnoreCase("http")))
                 .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find HTTP port"))
                 .getPort();
+        return port;
+    }
 
-        return "http://" + logServerHostInfo.getHostname() + ":" + port + "/logs";
+    /** Finds all hosts, grouping them by application ID and cluster name */
+    private  Map<ApplicationId, Map<String, List<URI>>> getHostsPerApplication() {
+        Map<ApplicationId, Map<String, List<URI>>> applicationHosts = new HashMap<>();
+        tenantRepository.getAllTenants().stream()
+                .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream())
+                .forEach(applicationId ->{
+                            applicationHosts.put(applicationId, getClustersOfApplication(applicationId));
+                        }
+                );
+        return applicationHosts;
     }
 
+    /** Finds the hosts of an application, grouped by cluster name */
+    private Map<String, List<URI>> getClustersOfApplication(ApplicationId applicationId) {
+        Application application = getApplication(applicationId);
+        Map<String, List<URI>> clusterHosts = new HashMap<>();
+        application.getModel().getHosts().stream()
+                .forEach(hostInfo -> {
+                            ServiceInfo serviceInfo = hostInfo.getServices().stream().filter(service -> METRICS_PROXY_CONTAINER.serviceName.equals(service.getServiceType()))
+                                    .findFirst().orElseThrow(() -> new IllegalArgumentException("Unable to find services " + METRICS_PROXY_CONTAINER.serviceName.toString()));
+                            String clusterName = serviceInfo.getProperty("clusterinfo").orElse("");
+                            URI host = URI.create("http://" + hostInfo.getHostname() + ":" + servicePort(serviceInfo) + "/metrics");
+                            clusterHosts.computeIfAbsent(clusterName, l -> new ArrayList<URI>()).add(host);
+                        }
+                );
+        return clusterHosts;
+
+    }
     /** Returns version to use when deploying application in given environment */
     static Version decideVersion(ApplicationId application, Environment environment, Version sessionVersion, boolean bootstrap) {
         if (     environment.isManuallyDeployed()
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index e04f83cc648..37f78dafa49 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -102,6 +102,10 @@ public class ApplicationHandler extends HttpHandler {
             return applicationRepository.getLogs(applicationId, hostname, apiParams);
         }
 
+        if (isMetricsRequest(request) && false) {
+            return applicationRepository.getMetrics();
+        }
+
         if (isIsSuspendedRequest(request)) {
             return new ApplicationSuspendedResponse(applicationRepository.isSuspended(applicationId));
         }
@@ -152,6 +156,7 @@ public class ApplicationHandler extends HttpHandler {
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/clustercontroller/*/status/*",
                 "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*",
                 "http://*/application/v2/tenant/*/application/*/logs",
+                "http://*/application/v2/metrics",
                 "http://*/application/v2/tenant/*/application/*");
     }
 
@@ -160,6 +165,11 @@ public class ApplicationHandler extends HttpHandler {
                request.getUri().getPath().endsWith("/suspended");
     }
 
+    private static boolean isMetricsRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 1 &&
+                request.getUri().getPath().endsWith("/metrics");
+    }
+
     private static boolean isLogRequest(HttpRequest request) {
         return getBindingMatch(request).groupCount() == 4 &&
                 request.getUri().getPath().endsWith("/logs");
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java
new file mode 100644
index 00000000000..4d6a2f20cd1
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/MetricsRespone.java
@@ -0,0 +1,53 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.http.v2;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.server.http.HttpConfigResponse;
+import com.yahoo.vespa.config.server.metrics.Metrics;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * @author olaa
+ */
+public class MetricsRespone extends HttpResponse {
+
+    private final Slime slime = new Slime();
+
+    public MetricsRespone(int status, Map<ApplicationId, Map<String, Metrics>> aggregatedMetrics) {
+        super(status);
+
+        Cursor array = slime.setArray();
+        for (Map.Entry<ApplicationId, Map<String, Metrics>> entry : aggregatedMetrics.entrySet()) {
+            Cursor object = array.addObject();
+            object.setString("applicationId", entry.getKey().serializedForm());
+            Cursor clusterArray = object.setArray("clusters");
+            for (Map.Entry<String, Metrics> clusterMetrics : entry.getValue().entrySet()) {
+                Cursor clusterCursor = clusterArray.addObject();
+                Metrics metrics = clusterMetrics.getValue();
+                clusterCursor.setString("clusterName", clusterMetrics.getKey());
+                clusterCursor.setDouble("queriesPerSecond", metrics.getQueriesPerSecond());
+                clusterCursor.setDouble("writesPerSecond", metrics.getWritesPerSecond());
+                clusterCursor.setDouble("documentCount", metrics.getDocumentCount());
+                clusterCursor.setDouble("queryLatencyMillis", metrics.getQueryLatencyMillis());
+                clusterCursor.setDouble("writeLatencyMillis", metrics.getWriteLatencyMills());
+            }
+        }
+    }
+
+    @Override
+    public void render(OutputStream outputStream) throws IOException {
+        new JsonFormat(false).encode(outputStream, slime);
+    }
+
+    @Override
+    public String getContentType() {
+        return HttpConfigResponse.JSON_CONTENT_TYPE;
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java
new file mode 100644
index 00000000000..7216523719a
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/Metrics.java
@@ -0,0 +1,63 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.metrics;
+
+import java.util.List;
+
+/**
+ * @author olaa
+ */
+public class Metrics {
+
+    private double queriesPerSecond;
+    private double writesPerSecond;
+    private double documentCount;
+    private double queryLatencyMillis;
+    private double writeLatencyMills;
+
+    public Metrics(double queriesPerSecond, double writesPerSecond, double documentCount,
+                             double queryLatencyMillis, double writeLatencyMills) {
+        this.queriesPerSecond = queriesPerSecond;
+        this.writesPerSecond = writesPerSecond;
+        this.documentCount = documentCount;
+        this.queryLatencyMillis = queryLatencyMillis;
+        this.writeLatencyMills = writeLatencyMills;
+    }
+
+
+    public double getQueriesPerSecond() {
+        return queriesPerSecond;
+    }
+
+    public double getWritesPerSecond() {
+        return writesPerSecond;
+    }
+
+    public double getDocumentCount() {
+        return documentCount;
+    }
+
+    public double getQueryLatencyMillis() {
+        return queryLatencyMillis;
+    }
+
+    public double getWriteLatencyMills() {
+        return writeLatencyMills;
+    }
+
+    public void accumulate(Metrics metrics) {
+        this.queriesPerSecond += metrics.getQueriesPerSecond();
+        this.writesPerSecond += metrics.getWritesPerSecond();
+        this.queryLatencyMillis += metrics.getQueryLatencyMillis();
+        this.writeLatencyMills += metrics.getWriteLatencyMills();
+    }
+
+    public static Metrics averagedMetrics(List<Metrics> metrics) {
+        return new Metrics(
+                metrics.stream().mapToDouble(Metrics::getQueriesPerSecond).sum() / metrics.size(),
+                metrics.stream().mapToDouble(Metrics::getWritesPerSecond).sum() / metrics.size(),
+                metrics.stream().mapToDouble(Metrics::getDocumentCount).sum() / metrics.size(),
+                metrics.stream().mapToDouble(Metrics::getQueryLatencyMillis).sum() / metrics.size(),
+                metrics.stream().mapToDouble(Metrics::getWriteLatencyMills).sum() / metrics.size()
+        );
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java
new file mode 100644
index 00000000000..bf1618b1696
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/MetricsAggregator.java
@@ -0,0 +1,82 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.metrics;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.config.server.http.v2.MetricsRespone;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClientBuilder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+
+/**
+ * @author olaa
+ */
+public class MetricsAggregator {
+
+    private static final Logger logger = Logger.getLogger(MetricsAggregator.class.getName());
+    HttpClient httpClient = HttpClientBuilder.create().build();
+
+    public MetricsRespone aggregateAllMetrics(Map<ApplicationId, Map<String, List<URI>>> applicationHosts) {
+        Map<ApplicationId, Map<String, Metrics>> aggregatedMetrics = applicationHosts.entrySet().stream()
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        e-> aggregateMetricsByCluster(e.getValue())));
+        return new MetricsRespone(200, aggregatedMetrics);
+    }
+
+    private Map<String, Metrics> aggregateMetricsByCluster(Map<String, List<URI>> clusterHosts) {
+        logger.log(Level.WARNING, clusterHosts.keySet() +" ");
+        return clusterHosts.entrySet().stream()
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        e -> aggregateMetrics(e.getValue())
+                )
+            );
+    }
+
+    private Metrics aggregateMetrics(List<URI> hosts) {
+        List<Metrics> metrics= hosts.stream()
+                .map(host -> getMetrics(host))
+                .collect(Collectors.toList());
+        Metrics.averagedMetrics(metrics);
+        logger.log(Level.WARNING, metrics + "");
+        return Metrics.averagedMetrics(metrics);
+    }
+
+    private Metrics getMetrics(URI hostURI) {
+        HttpGet get = new HttpGet(hostURI);
+        try {
+            HttpResponse response = httpClient.execute(get);
+
+            InputStream is = response.getEntity().getContent();
+            Slime slime = SlimeUtils.jsonToSlime(is.readAllBytes());
+            is.close();
+
+            Inspector metrics = slime.get().field("metrics");
+            Instant timestamp = Instant.ofEpochSecond(slime.get().field("timestamp").asLong());
+            double queriesPerSecond = metrics.field("queriesPerSecond").asDouble();
+            double writesPerSecond = metrics.field("writesPerSecond").asDouble();
+            double documentCount = metrics.field("documentCount").asDouble();
+            double queryLatencyMillis = metrics.field("queryLatencyMillis").asDouble();
+            double writeLatencyMills = metrics.field("writeLatencyMills").asDouble();
+            return new Metrics(queriesPerSecond, writesPerSecond, documentCount, queryLatencyMillis, writeLatencyMills);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index 23864e7d3e8..8ba505f213d 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -129,6 +129,7 @@
       <binding>http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*</binding>
       <binding>http://*/application/v2/tenant/*/application/*</binding>
       <binding>http://*/application/v2/tenant/*/application/*/logs</binding>
+      <binding>http://*/application/v2/metrics</binding>
     </handler>
     <handler id='com.yahoo.vespa.config.server.http.v2.HttpGetConfigHandler' bundle='configserver'>
       <binding>http://*/config/v2/tenant/*/application/*/*</binding>
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index a843212927b..f1891430ccd 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -35,6 +35,8 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.time.Clock;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java
new file mode 100644
index 00000000000..52c7cbec43e
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/MetricsAggregatorTest.java
@@ -0,0 +1,89 @@
+package com.yahoo.vespa.config.server.metrics;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.config.server.application.Application;
+import com.yahoo.vespa.config.server.application.TenantApplications;
+import com.yahoo.vespa.config.server.http.v2.MetricsRespone;
+import com.yahoo.vespa.config.server.tenant.Tenant;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;import com.yahoo.config.model.api.Model;
+
+
+/**
+ * @author olaa
+ */
+public class MetricsAggregatorTest {
+
+    @Rule
+    public final WireMockRule wireMock = new WireMockRule(options().port(8080), true);
+
+    @Test
+    public void testMetricAggregation() throws IOException {
+        MetricsAggregator metricsAggregator = new MetricsAggregator();
+
+        ApplicationId applicationId = ApplicationId.from("tenant", "app", "default");
+        Map<String, List<URI>> clusterHosts = Map.of(
+                "cluster1", List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/3")),
+                "cluster2", List.of(URI.create("http://localhost:8080/3"), URI.create("http://localhost:8080/3"))
+        );
+        Map<ApplicationId, Map<String, List<URI>>> applications = Map.of(applicationId, clusterHosts);
+
+        stubFor(get(urlEqualTo("/1"))
+                .willReturn(aResponse()
+                        .withStatus(200)
+                        .withBody(metricsString(10,20,33,40,50))));
+
+        stubFor(get(urlEqualTo("/2"))
+                .willReturn(aResponse()
+                        .withStatus(200)
+                        .withBody(metricsString(1,2,3,4,5))));
+
+        stubFor(get(urlEqualTo("/3"))
+                .willReturn(aResponse()
+                        .withStatus(200)
+                        .withBody(metricsString(1,2,3,4,5))));
+
+        MetricsRespone metricsRespone = metricsAggregator.aggregateAllMetrics(applications);
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        metricsRespone.render(bos);
+        assertEquals(Files.readString(Path.of("src/test/resources/metrics_response")), bos.toString());
+        wireMock.stop();
+    }
+
+    private String metricsString(double queriesPerSecond, double writesPerSecond, double documentCount, double queryLatencyMillis, double writeLatencyMills) {
+        return "{\"metrics\": " +
+                "{" +
+                "   \"queriesPerSecond\": " + queriesPerSecond + "," +
+                "   \"writesPerSecond\": " + writesPerSecond + "," +
+                "   \"documentCount\": " + documentCount + "," +
+                "   \"queryLatencyMillis\": " + queryLatencyMillis + "," +
+                "   \"writeLatencyMills\": " + writeLatencyMills +
+                "   }}";
+    }
+}
\ No newline at end of file
diff --git a/configserver/src/test/resources/metrics_response b/configserver/src/test/resources/metrics_response
new file mode 100644
index 00000000000..b400a0538a6
--- /dev/null
+++ b/configserver/src/test/resources/metrics_response
@@ -0,0 +1,23 @@
+[
+ {
+  "applicationId": "tenant:app:default",
+  "clusters": [
+   {
+    "clusterName": "cluster1",
+    "queriesPerSecond": 5.5,
+    "writesPerSecond": 11.0,
+    "documentCount": 18.0,
+    "queryLatencyMillis": 22.0,
+    "writeLatencyMillis": 27.5
+   },
+   {
+    "clusterName": "cluster2",
+    "queriesPerSecond": 1.0,
+    "writesPerSecond": 2.0,
+    "documentCount": 3.0,
+    "queryLatencyMillis": 4.0,
+    "writeLatencyMillis": 5.0
+   }
+  ]
+ }
+]
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 0cf89d798a7..dd390daaeb0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -97,6 +97,7 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
         }
+        newMaintain();
     }
 
     /** Get global rotation status for application */
@@ -120,4 +121,14 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         }
     }
 
+    private void newMaintain() {
+        controller().zoneRegistry()
+                .zones()
+                .reachable()
+                .ids()
+                .stream()
+                .forEach(zoneId -> {
+
+                });
+    }
 }