Merge pull request #10563 from vespa-engine/mpolden/move-configsrv-metrics

Move config server metrics fetching to controller-server

16 files changed, 294 insertions, 108 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index c29e64c4cd7..77b2f604d5d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -9,9 +9,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.ApplicationActivity;
@@ -21,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Instant;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 3a55b926850..8ab3d4b6299 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -13,13 +13,13 @@ import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry;
 import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.api.integration.user.Roles;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.TenantRole;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger;
+import com.yahoo.vespa.hosted.controller.metric.ConfigServerMetrics;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -68,7 +68,7 @@ public class Controller extends AbstractComponent {
     private final Clock clock;
     private final ZoneRegistry zoneRegistry;
     private final ServiceRegistry serviceRegistry;
-    private final MetricsService metricsService;
+    private final ConfigServerMetrics metrics;
     private final AuditLogger auditLogger;
     private final FlagSource flagSource;
     private final NameServiceForwarder nameServiceForwarder;
@@ -81,13 +81,12 @@ public class Controller extends AbstractComponent {
      */
     @Inject
     public Controller(CuratorDb curator, RotationsConfig rotationsConfig,
-                      ZoneRegistry zoneRegistry, MetricsService metricsService,
+                      ZoneRegistry zoneRegistry,
                       AccessControl accessControl,
                       FlagSource flagSource,
                       MavenRepository mavenRepository,
                       ServiceRegistry serviceRegistry) {
         this(curator, rotationsConfig, zoneRegistry,
-             metricsService,
              Clock.systemUTC(), accessControl,
              com.yahoo.net.HostName::getLocalhost, flagSource,
              mavenRepository, serviceRegistry);
@@ -95,7 +94,6 @@ public class Controller extends AbstractComponent {
 
     public Controller(CuratorDb curator, RotationsConfig rotationsConfig,
                       ZoneRegistry zoneRegistry,
-                      MetricsService metricsService,
                       Clock clock,
                       AccessControl accessControl,
                       Supplier<String> hostnameSupplier,
@@ -106,11 +104,12 @@ public class Controller extends AbstractComponent {
         this.curator = Objects.requireNonNull(curator, "Curator cannot be null");
         this.zoneRegistry = Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null");
         this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null");
-        this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null");
         this.clock = Objects.requireNonNull(clock, "Clock cannot be null");
         this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null");
         this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null");
 
+
+        metrics = new ConfigServerMetrics(serviceRegistry.configServer());
         nameServiceForwarder = new NameServiceForwarder(curator);
         jobController = new JobController(this);
         applicationController = new ApplicationController(this, curator, accessControl,
@@ -256,8 +255,8 @@ public class Controller extends AbstractComponent {
         return HostName.from(hostnameSupplier.get());
     }
 
-    public MetricsService metricsService() {
-        return metricsService;
+    public ConfigServerMetrics metrics() {
+        return metrics;
     }
 
     public SystemName system() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index ff7d5142f1d..d1a5d7bbf28 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -8,11 +8,8 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
@@ -23,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Instant;
@@ -245,7 +243,7 @@ public class LockedApplication {
                                      metrics, pemDeployKey, rotations, rotationStatus);
     }
 
-    public LockedApplication with(MetricsService.ApplicationMetrics metrics) {
+    public LockedApplication with(ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
                                      metrics, pemDeployKey, rotations, rotationStatus);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 3927630abba..7b925ab7691 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -5,12 +5,10 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 
 import java.time.Duration;
-import java.time.Instant;
 import java.util.List;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
@@ -51,13 +49,12 @@ public class DeploymentMetricsMaintainer extends Maintainer {
             applicationList.parallelStream().forEach(application -> {
                 try {
                     applications.lockIfPresent(application.id(), locked ->
-                            applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id()))));
+                            applications.store(locked.with(controller().metrics().getApplicationMetrics(application.id()))));
 
                     for (Deployment deployment : application.deployments().values()) {
                         if (deployment.version().getMajor() < 7) continue;
-                        MetricsService.DeploymentMetrics collectedMetrics = controller().metricsService()
-                                                                                        .getDeploymentMetrics(application.id(), deployment.zone());
-                        Instant now = controller().clock().instant();
+                        var collectedMetrics = controller().metrics().getDeploymentMetrics(application.id(), deployment.zone());
+                        var now = controller().clock().instant();
                         applications.lockIfPresent(application.id(), locked -> {
                             Deployment existingDeployment = locked.get().deployments().get(deployment.zone());
                             if (existingDeployment == null) return; // Deployment removed since we started collecting metrics
@@ -94,8 +91,4 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         }
     }
 
-    private static DeploymentMetricsMaintainer maintainer(Controller controller) {
-        return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java
new file mode 100644
index 00000000000..904ade916b1
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java
@@ -0,0 +1,33 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.metric;
+
+/**
+ * Application metrics aggregated across all deployments.
+ *
+ * @author bratseth
+ */
+public class ApplicationMetrics {
+
+    private final double queryServiceQuality;
+    private final double writeServiceQuality;
+
+    public ApplicationMetrics(double queryServiceQuality, double writeServiceQuality) {
+        this.queryServiceQuality = queryServiceQuality;
+        this.writeServiceQuality = writeServiceQuality;
+    }
+
+    /**
+     * Returns the quality of service for queries as a number between 1 (perfect) and 0 (none)
+     */
+    public double queryServiceQuality() {
+        return queryServiceQuality;
+    }
+
+    /**
+     * Returns the quality of service for writes as a number between 1 (perfect) and 0 (none)
+     */
+    public double writeServiceQuality() {
+        return writeServiceQuality;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java
new file mode 100644
index 00000000000..07cf7045e66
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java
@@ -0,0 +1,64 @@
+package com.yahoo.vespa.hosted.controller.metric;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+
+/**
+ * Retrieves metrics from the configuration server.
+ *
+ * @author ogronnesby
+ */
+public class ConfigServerMetrics {
+
+    private final ConfigServer configServer;
+
+    public ConfigServerMetrics(ConfigServer configServer) {
+        this.configServer = configServer;
+    }
+
+    public ApplicationMetrics getApplicationMetrics(ApplicationId application) {
+        // TODO(ogronnesby): How to produce these values in Public context?
+        return new ApplicationMetrics(0.0, 0.0);
+    }
+
+    public DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone) {
+        var deploymentId = new DeploymentId(application, zone);
+        var metrics = configServer.getMetrics(deploymentId);
+
+        // The field names here come from the MetricsResponse class.
+        return new DeploymentMetrics(
+                metrics.stream().flatMap(m -> m.queriesPerSecond().stream()).mapToDouble(Double::doubleValue).sum(),
+                metrics.stream().flatMap(m -> m.feedPerSecond().stream()).mapToDouble(Double::doubleValue).sum(),
+                metrics.stream().flatMap(m -> m.documentCount().stream()).mapToLong(Double::longValue).sum(),
+                weightedAverageLatency(metrics, ClusterMetrics::queriesPerSecond, ClusterMetrics::queryLatency),
+                weightedAverageLatency(metrics, ClusterMetrics::feedPerSecond, ClusterMetrics::feedLatency)
+        );
+    }
+
+    private double weightedAverageLatency(List<ClusterMetrics> metrics,
+                                          Function<ClusterMetrics, Optional<Double>> rateExtractor,
+                                          Function<ClusterMetrics, Optional<Double>> latencyExtractor)
+    {
+        var rateSum = metrics.stream().flatMap(m -> rateExtractor.apply(m).stream()).mapToDouble(Double::longValue).sum();
+        if (rateSum == 0) {
+            return 0.0;
+        }
+
+        var weightedLatency = metrics.stream()
+                .flatMap(m -> {
+                    return latencyExtractor.apply(m).flatMap(l -> rateExtractor.apply(m).map(r -> l * r)).stream();
+                })
+                .mapToDouble(Double::doubleValue)
+                .sum();
+
+        return weightedLatency / rateSum;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/DeploymentMetrics.java
new file mode 100644
index 00000000000..33a3ce957ed
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/DeploymentMetrics.java
@@ -0,0 +1,47 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.metric;
+
+/**
+ * Metrics for a single deployment of an application.
+ *
+ * @author bratseth
+ */
+public class DeploymentMetrics {
+
+    private final double queriesPerSecond;
+    private final double writesPerSecond;
+    private final long documentCount;
+    private final double queryLatencyMillis;
+    private final double writeLatencyMillis;
+
+    public DeploymentMetrics(double queriesPerSecond, double writesPerSecond,
+                             long documentCount,
+                             double queryLatencyMillis, double writeLatencyMillis) {
+        this.queriesPerSecond = queriesPerSecond;
+        this.writesPerSecond = writesPerSecond;
+        this.documentCount = documentCount;
+        this.queryLatencyMillis = queryLatencyMillis;
+        this.writeLatencyMillis = writeLatencyMillis;
+    }
+
+    public double queriesPerSecond() {
+        return queriesPerSecond;
+    }
+
+    public double writesPerSecond() {
+        return writesPerSecond;
+    }
+
+    public long documentCount() {
+        return documentCount;
+    }
+
+    public double queryLatencyMillis() {
+        return queryLatencyMillis;
+    }
+
+    public double writeLatencyMillis() {
+        return writeLatencyMillis;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index d2a27e0c76d..54dba6ec0db 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
@@ -30,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index f8bd1343436..29a00784e6d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -25,14 +25,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
-import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
-import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
 import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
 import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer;
@@ -72,33 +70,28 @@ public final class ControllerTester {
     private final ServiceRegistryMock serviceRegistry;
     private final CuratorDb curator;
     private final RotationsConfig rotationsConfig;
-    private final MockBuildService buildService;
-    private final MetricsServiceMock metricsService;
 
     private Controller controller;
 
-    public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb,
-                            MetricsServiceMock metricsService) {
+    public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb) {
         this(new AthenzDbMock(),
              clock,
              new ZoneRegistryMock(),
              curatorDb,
              rotationsConfig,
-             new MockBuildService(),
-             metricsService,
              new ServiceRegistryMock());
     }
 
     public ControllerTester(ManualClock clock) {
-        this(clock, defaultRotationsConfig(), new MockCuratorDb(), new MetricsServiceMock());
+        this(clock, defaultRotationsConfig(), new MockCuratorDb());
     }
 
     public ControllerTester(RotationsConfig rotationsConfig) {
-        this(new ManualClock(), rotationsConfig, new MockCuratorDb(), new MetricsServiceMock());
+        this(new ManualClock(), rotationsConfig, new MockCuratorDb());
     }
 
     public ControllerTester(MockCuratorDb curatorDb) {
-        this(new ManualClock(), defaultRotationsConfig(), curatorDb, new MetricsServiceMock());
+        this(new ManualClock(), defaultRotationsConfig(), curatorDb);
     }
 
     public ControllerTester() {
@@ -108,8 +101,6 @@ public final class ControllerTester {
     private ControllerTester(AthenzDbMock athenzDb, ManualClock clock,
                              ZoneRegistryMock zoneRegistry,
                              CuratorDb curator, RotationsConfig rotationsConfig,
-                             MockBuildService buildService,
-                             MetricsServiceMock metricsService,
                              ServiceRegistryMock serviceRegistry) {
         this.athenzDb = athenzDb;
         this.clock = clock;
@@ -117,11 +108,7 @@ public final class ControllerTester {
         this.serviceRegistry = serviceRegistry;
         this.curator = curator;
         this.rotationsConfig = rotationsConfig;
-        this.buildService = buildService;
-        this.metricsService = metricsService;
-        this.controller = createController(curator, rotationsConfig, clock, zoneRegistry,
-                                           athenzDb,
-                                           metricsService, serviceRegistry);
+        this.controller = createController(curator, rotationsConfig, clock, zoneRegistry, athenzDb, serviceRegistry);
 
         // Make root logger use time from manual clock
         configureDefaultLogHandler(handler -> handler.setFilter(
@@ -159,8 +146,6 @@ public final class ControllerTester {
 
     public ServiceRegistryMock serviceRegistry() { return serviceRegistry; }
 
-    public MetricsServiceMock metricsService() { return metricsService; }
-
     public Optional<Record> findCname(String name) {
         return serviceRegistry.nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream().findFirst();
     }
@@ -168,7 +153,6 @@ public final class ControllerTester {
     /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */
     public final void createNewController() {
         controller = createController(curator, rotationsConfig, clock, zoneRegistry, athenzDb,
-                                      metricsService,
                                       serviceRegistry);
     }
 
@@ -305,12 +289,10 @@ public final class ControllerTester {
                                                ManualClock clock,
                                                ZoneRegistryMock zoneRegistryMock,
                                                AthenzDbMock athensDb,
-                                               MetricsServiceMock metricsService,
                                                ServiceRegistryMock serviceRegistry) {
         Controller controller = new Controller(curator,
                                                rotationsConfig,
                                                zoneRegistryMock,
-                                               metricsService,
                                                clock,
                                                new AthenzFacade(new AthenzClientFactoryMock(athensDb)),
                                                () -> "test-controller",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 4d16b546d6b..4a7ee8bcb63 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -64,6 +64,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>();
     private final Map<DeploymentId, List<Log>> warnings = new HashMap<>();
     private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>();
+    private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>();
 
     private Version lastPrepareVersion = null;
     private RuntimeException prepareException = null;
@@ -188,6 +189,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return Collections.unmodifiableMap(rotationNames);
     }
 
+    public void setMetrics(DeploymentId deployment, ClusterMetrics clusterMetrics) {
+        setMetrics(deployment, List.of(clusterMetrics));
+    }
+
+    public void setMetrics(DeploymentId deployment, List<ClusterMetrics> clusterMetrics) {
+        this.clusterMetrics.put(deployment, clusterMetrics);
+    }
+
     @Override
     public NodeRepositoryMock nodeRepository() {
         return nodeRepository;
@@ -336,7 +345,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
     @Override
     public List<ClusterMetrics> getMetrics(DeploymentId deployment) {
-        return List.of();
+        return Collections.unmodifiableList(clusterMetrics.getOrDefault(deployment, List.of()));
     }
 
     // Returns a canned example response
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java
deleted file mode 100644
index 2eae50cbf9f..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.integration;
-
-import com.yahoo.component.AbstractComponent;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @author bratseth
- */
-public class MetricsServiceMock extends AbstractComponent implements MetricsService {
-
-    private final Map<String, Double> metrics = new HashMap<>();
-
-    public MetricsServiceMock setMetric(String key, Double value) {
-        metrics.put(key, value);
-        return this;
-    }
-
-    @Override
-    public ApplicationMetrics getApplicationMetrics(ApplicationId application) {
-        return new ApplicationMetrics(metrics.getOrDefault("queryServiceQuality", 0.5),
-                                      metrics.getOrDefault("writeServiceQuality", 0.7));
-    }
-
-    @Override
-    public DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone) {
-        return new DeploymentMetrics(metrics.getOrDefault("queriesPerSecond", 1D),
-                                     metrics.getOrDefault("writesPerSecond", 2D),
-                                     metrics.getOrDefault("docoumentCount", 3D).longValue(),
-                                     metrics.getOrDefault("queryLatencyMillis", 4D),
-                                     metrics.getOrDefault("writeLatencyMillis", 5D));
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index 424cb05ebef..c8b3cd01fe8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -8,13 +8,17 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import org.junit.Test;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.function.Supplier;
 
@@ -28,14 +32,15 @@ import static org.junit.Assert.assertFalse;
  */
 public class DeploymentMetricsMaintainerTest {
 
-    ControllerTester tester = new ControllerTester();
+    private final DeploymentTester tester = new DeploymentTester();
 
     @Test
     public void updates_metrics() {
-        ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1",
-                                                  Environment.dev, 123).id();
+        var application = tester.createApplication("app1", "tenant1", 123L, 1L);
+        deploy(application, Version.fromString("7.1"));
+
         DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
-        Supplier<Application> app = tester.application(appId);
+        Supplier<Application> app = () -> tester.application(application.id());
         Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get();
 
         // No metrics gathered yet
@@ -48,7 +53,7 @@ public class DeploymentMetricsMaintainerTest {
         // Only get application metrics for old version
         deploy(app.get(), Version.fromString("6.3.3"));
         maintainer.maintain();
-        assertEquals(0.5, app.get().metrics().queryServiceQuality(), 0);
+        assertEquals(0, app.get().metrics().queryServiceQuality(), 0);
         assertEquals(0, deployment.get().metrics().documentCount(), 0);
         assertFalse("No timestamp set", deployment.get().metrics().instant().isPresent());
         assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent());
@@ -56,10 +61,16 @@ public class DeploymentMetricsMaintainerTest {
 
         // Metrics are gathered and saved to application
         deploy(app.get(), Version.fromString("7.5.5"));
+        var metrics0 = Map.of(ClusterMetrics.QUERIES_PER_SECOND, 1D,
+                              ClusterMetrics.FEED_PER_SECOND, 2D,
+                              ClusterMetrics.DOCUMENT_COUNT, 3D,
+                              ClusterMetrics.QUERY_LATENCY, 4D,
+                              ClusterMetrics.FEED_LATENCY, 5D);
+        setMetrics(application.id(), metrics0);
         maintainer.maintain();
         Instant t1 = tester.clock().instant().truncatedTo(MILLIS);
-        assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE);
-        assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE);
+        assertEquals(0.0, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE);
+        assertEquals(0.0, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE);
         assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE);
         assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE);
         assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE);
@@ -82,8 +93,10 @@ public class DeploymentMetricsMaintainerTest {
         // Query traffic disappears. Query activity stops updating
         tester.clock().advance(Duration.ofHours(1));
         Instant t3 = tester.clock().instant().truncatedTo(MILLIS);
-        tester.metricsService().setMetric("queriesPerSecond", 0D);
-        tester.metricsService().setMetric("writesPerSecond", 5D);
+        var metrics1 = new HashMap<>(metrics0);
+        metrics1.put(ClusterMetrics.QUERIES_PER_SECOND, 0D);
+        metrics1.put(ClusterMetrics.FEED_PER_SECOND, 5D);
+        setMetrics(application.id(), metrics1);
         maintainer.maintain();
         assertEquals(t2, deployment.get().activity().lastQueried().get());
         assertEquals(t3, deployment.get().activity().lastWritten().get());
@@ -92,7 +105,9 @@ public class DeploymentMetricsMaintainerTest {
 
         // Feed traffic disappears. Feed activity stops updating
         tester.clock().advance(Duration.ofHours(1));
-        tester.metricsService().setMetric("writesPerSecond", 0D);
+        var metrics2 = new HashMap<>(metrics1);
+        metrics2.put(ClusterMetrics.FEED_PER_SECOND, 0D);
+        setMetrics(application.id(), metrics2);
         maintainer.maintain();
         assertEquals(t2, deployment.get().activity().lastQueried().get());
         assertEquals(t3, deployment.get().activity().lastWritten().get());
@@ -100,15 +115,24 @@ public class DeploymentMetricsMaintainerTest {
         assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
     }
 
+    private void setMetrics(ApplicationId application, Map<String, Double> metrics) {
+        var clusterMetrics = new ClusterMetrics("default", ClusterMetrics.ClusterType.container);
+        for (var kv : metrics.entrySet()) {
+            clusterMetrics = clusterMetrics.addMetric(kv.getKey(), kv.getValue());
+        }
+        tester.controllerTester().serviceRegistry().configServerMock().setMetrics(new DeploymentId(application, ZoneId.from("dev", "us-east-1")), clusterMetrics);
+    }
+
     private static DeploymentMetricsMaintainer maintainer(Controller controller) {
         return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
     }
 
     private void deploy(Application application, Version version) {
-        tester.deploy(application,
-                ZoneId.from(Environment.dev, RegionName.from("us-east-1")),
-                Optional.of(new ApplicationPackage(new byte[0])),
-                false,
-                Optional.of(version));
+        tester.controllerTester().deploy(application,
+                                         ZoneId.from(Environment.dev, RegionName.from("us-east-1")),
+                                         Optional.of(new ApplicationPackage(new byte[0])),
+                                         false,
+                                         Optional.of(version));
     }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java
new file mode 100644
index 00000000000..23a6c6286c0
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java
@@ -0,0 +1,78 @@
+package com.yahoo.vespa.hosted.controller.metric;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author olaa
+ */
+public class ConfigServerMetricsTest {
+
+    private final ApplicationId applicationId = new ApplicationId.Builder()
+                .tenant("foo")
+                .applicationName("bar")
+                .instanceName("default")
+                .build();
+
+    private final ZoneId zoneId = ZoneId.from("prod", "us-west-1");
+
+    private ConfigServerMock configServer;
+    private ConfigServerMetrics service;
+
+    @Before
+    public void before() {
+        configServer = new ConfigServerMock(new ZoneRegistryMock());
+        service = new ConfigServerMetrics(configServer);
+    }
+
+    @Test
+    public void test_returning_metrics() {
+        //
+        // Wire up the test
+        //
+        var deploymentId = new DeploymentId(applicationId, zoneId);
+
+        var clusterMetrics1 = new ClusterMetrics("niceCluster", ClusterMetrics.ClusterType.container) {{
+            addMetric("queriesPerSecond", 23.0);
+            addMetric("queryLatency", 1337.0);
+        }};
+
+        var clusterMetrics2 = new ClusterMetrics("alsoNiceCluster", ClusterMetrics.ClusterType.container) {{
+            addMetric("queriesPerSecond", 11.0);
+            addMetric("queryLatency", 12.0);
+        }};
+
+        var response = List.of(clusterMetrics1, clusterMetrics2);
+
+        configServer.setMetrics(deploymentId, response);
+
+        //
+        // Now we can actually test stuff :(
+        //
+        var deploymentMetrics = service.getDeploymentMetrics(applicationId, zoneId);
+
+        assertEquals(23.0 + 11.0, deploymentMetrics.queriesPerSecond(), 0.001);
+        assertEquals(908.323, deploymentMetrics.queryLatencyMillis(), 0.001);
+        assertEquals(0, deploymentMetrics.documentCount());
+        assertEquals(0.0, deploymentMetrics.writeLatencyMillis(), 0.001);
+        assertEquals(0.0, deploymentMetrics.writesPerSecond(), 0.001);
+    }
+
+    @Test
+    public void test_not_implemented_application_metrics() {
+        var applicationMetrics = service.getApplicationMetrics(applicationId);
+        assertEquals(0.0, applicationMetrics.queryServiceQuality(), 0.001);
+        assertEquals(0.0, applicationMetrics.writeServiceQuality(), 0.001);
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index d0e18be270a..16f57a61916 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
@@ -25,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
@@ -114,7 +114,7 @@ public class ApplicationSerializerTest {
                                                Optional.of(IssueId.from("1234")),
                                                Optional.of(User.from("by-username")),
                                                OptionalInt.of(7),
-                                               new MetricsService.ApplicationMetrics(0.5, 0.9),
+                                               new ApplicationMetrics(0.5, 0.9),
                                                Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"),
                                                List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of())),
                                                rotationStatus);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index 0fc7bcfc76f..0cfb79a0743 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -68,7 +68,6 @@ public class ControllerContainerTest {
                "  <component id='com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/>\n" +
-               "  <component id='com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository'/>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index ec6c905b7e7..c6cd388d75f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -31,7 +31,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
@@ -60,6 +59,7 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater;
+import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;