diff options
author | Martin Polden <mpolden@mpolden.no> | 2019-09-09 13:05:10 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2019-09-09 14:46:09 +0200 |
commit | 46708d8c64baea341aa332f18d026dd7936bc745 (patch) | |
tree | 490489429d4e92cf6be6ccc8eaf769c50873ffa2 | |
parent | dd25c2a3c4c16ba96eda61e996f5347f1f8eaa9c (diff) |
Move config server metrics fetching to controller-server
`MetricsService` interface and component setup is no longer needed.
19 files changed, 183 insertions, 260 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java index 28c94960daf..9296e144bf7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ClusterMetrics.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.OptionalDouble; /** * @author olaa @@ -12,11 +11,11 @@ import java.util.OptionalDouble; public class ClusterMetrics { // These field names originate from the MetricsResponse class - private static final String QUERIES_PER_SECOND = "queriesPerSecond"; - private static final String FEED_PER_SECOND = "feedPerSecond"; - private static final String DOCUMENT_COUNT = "documentCount"; - private static final String FEED_LATENCY = "feedLatency"; - private static final String QUERY_LATENCY = "queryLatency"; + public static final String QUERIES_PER_SECOND = "queriesPerSecond"; + public static final String FEED_PER_SECOND = "feedPerSecond"; + public static final String DOCUMENT_COUNT = "documentCount"; + public static final String FEED_LATENCY = "feedLatency"; + public static final String QUERY_LATENCY = "queryLatency"; private final String clusterId; private final ClusterType clusterType; @@ -56,9 +55,14 @@ public class ClusterMetrics { return Optional.ofNullable(metrics.get(QUERY_LATENCY)); } - public void addMetric(String name, double value) { + public ClusterMetrics addMetric(String name, double value) { metrics.put(name, value); + return this; + } + + public enum ClusterType { + content, + container } - public enum ClusterType {content, container}; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/MetricsService.java deleted file mode 100644 index abc9d53d4c7..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/MetricsService.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.metrics; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; - -/** - * A service which returns metric values on request - * - * @author bratseth - */ -public interface MetricsService { - - ApplicationMetrics getApplicationMetrics(ApplicationId application); - - DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone); - - class DeploymentMetrics { - - private final double queriesPerSecond; - private final double writesPerSecond; - private final long documentCount; - private final double queryLatencyMillis; - private final double writeLatencyMillis; - - public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, - long documentCount, - double queryLatencyMillis, double writeLatencyMillis) { - this.queriesPerSecond = queriesPerSecond; - this.writesPerSecond = writesPerSecond; - this.documentCount = documentCount; - this.queryLatencyMillis = queryLatencyMillis; - this.writeLatencyMillis = writeLatencyMillis; - } - - public double queriesPerSecond() { return queriesPerSecond; } - - public double writesPerSecond() { return writesPerSecond; } - - public long documentCount() { return documentCount; } - - public double queryLatencyMillis() { return queryLatencyMillis; } - - public double writeLatencyMillis() { return writeLatencyMillis; } - - } - - class ApplicationMetrics { - - private final double queryServiceQuality; - private final double writeServiceQuality; - - public ApplicationMetrics(double queryServiceQuality, double writeServiceQuality) { - this.queryServiceQuality = queryServiceQuality; - this.writeServiceQuality = writeServiceQuality; - } - - /** Returns the quality of service for queries as a number between 1 (perfect) and 0 (none) */ - public double queryServiceQuality() { return queryServiceQuality; } - - /** Returns the quality of service for writes as a number between 1 (perfect) and 0 (none) */ - public double writeServiceQuality() { return writeServiceQuality; } - - } - - class SystemMetrics { - - private final double cpuUtil; - private final double memUtil; - private final double diskUtil; - - /** - * @param cpuUtil percentage of system cpu utilization - * @param memUtil percentage of system memory utilization - * @param diskUtil percentage of system disk utilization - */ - public SystemMetrics(double cpuUtil, double memUtil, double diskUtil) { - this.cpuUtil = cpuUtil; - this.memUtil = memUtil; - this.diskUtil = diskUtil; - } - - /** @return the percentage of cpu utilization **/ - public double cpuUtil() { return cpuUtil; } - - /** @return the percentage of memory utilization **/ - public double memUtil() { return memUtil; } - - /** @return the percentage of disk utilization **/ - public double diskUtil() { return diskUtil; } - - public static class Builder { - private double cpuUtil; - private double memUtil; - private double diskUtil; - - public void setCpuUtil(double cpuUtil) { - this.cpuUtil = cpuUtil; - } - - public void setMemUtil(double memUtil) { - this.memUtil = memUtil; - } - - public void setDiskUtil(double diskUtil) { - this.diskUtil = diskUtil; - } - - public SystemMetrics build() { return new SystemMetrics(cpuUtil, memUtil, diskUtil); } - } - - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/package-info.java deleted file mode 100644 index d14f2e79882..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.api.integration.metrics; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index c29e64c4cd7..77b2f604d5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -9,9 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; @@ -21,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 3a55b926850..8ab3d4b6299 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -13,13 +13,13 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.TenantRole; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; +import com.yahoo.vespa.hosted.controller.metric.ConfigServerMetrics; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -68,7 +68,7 @@ public class Controller extends AbstractComponent { private final Clock clock; private final ZoneRegistry zoneRegistry; private final ServiceRegistry serviceRegistry; - private final MetricsService metricsService; + private final ConfigServerMetrics metrics; private final AuditLogger auditLogger; private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; @@ -81,13 +81,12 @@ public class Controller extends AbstractComponent { */ @Inject public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - ZoneRegistry zoneRegistry, MetricsService metricsService, + ZoneRegistry zoneRegistry, AccessControl accessControl, FlagSource flagSource, MavenRepository mavenRepository, ServiceRegistry serviceRegistry) { this(curator, rotationsConfig, zoneRegistry, - metricsService, Clock.systemUTC(), accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, mavenRepository, serviceRegistry); @@ -95,7 +94,6 @@ public class Controller extends AbstractComponent { public Controller(CuratorDb curator, RotationsConfig rotationsConfig, ZoneRegistry zoneRegistry, - MetricsService metricsService, Clock clock, AccessControl accessControl, Supplier<String> hostnameSupplier, @@ -106,11 +104,12 @@ public class Controller extends AbstractComponent { this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); this.zoneRegistry = Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); - this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null"); this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); + + metrics = new ConfigServerMetrics(serviceRegistry.configServer()); nameServiceForwarder = new NameServiceForwarder(curator); jobController = new JobController(this); applicationController = new ApplicationController(this, curator, accessControl, @@ -256,8 +255,8 @@ public class Controller extends AbstractComponent { return HostName.from(hostnameSupplier.get()); } - public MetricsService metricsService() { - return metricsService; + public ConfigServerMetrics metrics() { + return metrics; } public SystemName system() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index ff7d5142f1d..d1a5d7bbf28 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -8,11 +8,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; @@ -23,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; import java.time.Instant; @@ -245,7 +243,7 @@ public class LockedApplication { metrics, pemDeployKey, rotations, rotationStatus); } - public LockedApplication with(MetricsService.ApplicationMetrics metrics) { + public LockedApplication with(ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, rotations, rotationStatus); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 3927630abba..7b925ab7691 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -5,12 +5,10 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import java.time.Duration; -import java.time.Instant; import java.util.List; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; @@ -51,13 +49,12 @@ public class DeploymentMetricsMaintainer extends Maintainer { applicationList.parallelStream().forEach(application -> { try { applications.lockIfPresent(application.id(), locked -> - applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id())))); + applications.store(locked.with(controller().metrics().getApplicationMetrics(application.id())))); for (Deployment deployment : application.deployments().values()) { if (deployment.version().getMajor() < 7) continue; - MetricsService.DeploymentMetrics collectedMetrics = controller().metricsService() - .getDeploymentMetrics(application.id(), deployment.zone()); - Instant now = controller().clock().instant(); + var collectedMetrics = controller().metrics().getDeploymentMetrics(application.id(), deployment.zone()); + var now = controller().clock().instant(); applications.lockIfPresent(application.id(), locked -> { Deployment existingDeployment = locked.get().deployments().get(deployment.zone()); if (existingDeployment == null) return; // Deployment removed since we started collecting metrics @@ -94,8 +91,4 @@ public class DeploymentMetricsMaintainer extends Maintainer { } } - private static DeploymentMetricsMaintainer maintainer(Controller controller) { - return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator())); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java new file mode 100644 index 00000000000..904ade916b1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ApplicationMetrics.java @@ -0,0 +1,33 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.metric; + +/** + * Application metrics aggregated across all deployments. + * + * @author bratseth + */ +public class ApplicationMetrics { + + private final double queryServiceQuality; + private final double writeServiceQuality; + + public ApplicationMetrics(double queryServiceQuality, double writeServiceQuality) { + this.queryServiceQuality = queryServiceQuality; + this.writeServiceQuality = writeServiceQuality; + } + + /** + * Returns the quality of service for queries as a number between 1 (perfect) and 0 (none) + */ + public double queryServiceQuality() { + return queryServiceQuality; + } + + /** + * Returns the quality of service for writes as a number between 1 (perfect) and 0 (none) + */ + public double writeServiceQuality() { + return writeServiceQuality; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/ConfigServerMetricsService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java index 125212d7a65..07cf7045e66 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/ConfigServerMetricsService.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetrics.java @@ -1,11 +1,9 @@ -package com.yahoo.vespa.hosted.controller.api.integration.metrics; +package com.yahoo.vespa.hosted.controller.metric; -import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import java.util.List; @@ -17,30 +15,22 @@ import java.util.function.Function; * * @author ogronnesby */ -// TODO: This module should not contain components. Move this to controller-server. -public class ConfigServerMetricsService implements MetricsService { +public class ConfigServerMetrics { - private final ConfigServer configServerClient; + private final ConfigServer configServer; - @Inject - public ConfigServerMetricsService(ServiceRegistry serviceRegistry) { - this(serviceRegistry.configServer()); + public ConfigServerMetrics(ConfigServer configServer) { + this.configServer = configServer; } - ConfigServerMetricsService(ConfigServer configServer) { - this.configServerClient = configServer; - } - - @Override public ApplicationMetrics getApplicationMetrics(ApplicationId application) { // TODO(ogronnesby): How to produce these values in Public context? return new ApplicationMetrics(0.0, 0.0); } - @Override public DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone) { var deploymentId = new DeploymentId(application, zone); - var metrics = configServerClient.getMetrics(deploymentId); + var metrics = configServer.getMetrics(deploymentId); // The field names here come from the MetricsResponse class. return new DeploymentMetrics( @@ -70,4 +60,5 @@ public class ConfigServerMetricsService implements MetricsService { return weightedLatency / rateSum; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/DeploymentMetrics.java new file mode 100644 index 00000000000..33a3ce957ed --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/DeploymentMetrics.java @@ -0,0 +1,47 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.metric; + +/** + * Metrics for a single deployment of an application. + * + * @author bratseth + */ +public class DeploymentMetrics { + + private final double queriesPerSecond; + private final double writesPerSecond; + private final long documentCount; + private final double queryLatencyMillis; + private final double writeLatencyMillis; + + public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, + long documentCount, + double queryLatencyMillis, double writeLatencyMillis) { + this.queriesPerSecond = queriesPerSecond; + this.writesPerSecond = writesPerSecond; + this.documentCount = documentCount; + this.queryLatencyMillis = queryLatencyMillis; + this.writeLatencyMillis = writeLatencyMillis; + } + + public double queriesPerSecond() { + return queriesPerSecond; + } + + public double writesPerSecond() { + return writesPerSecond; + } + + public long documentCount() { + return documentCount; + } + + public double queryLatencyMillis() { + return queryLatencyMillis; + } + + public double writeLatencyMillis() { + return writeLatencyMillis; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index d2a27e0c76d..54dba6ec0db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; @@ -30,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index f8bd1343436..29a00784e6d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -25,14 +25,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; -import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; @@ -72,33 +70,28 @@ public final class ControllerTester { private final ServiceRegistryMock serviceRegistry; private final CuratorDb curator; private final RotationsConfig rotationsConfig; - private final MockBuildService buildService; - private final MetricsServiceMock metricsService; private Controller controller; - public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb, - MetricsServiceMock metricsService) { + public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb) { this(new AthenzDbMock(), clock, new ZoneRegistryMock(), curatorDb, rotationsConfig, - new MockBuildService(), - metricsService, new ServiceRegistryMock()); } public ControllerTester(ManualClock clock) { - this(clock, defaultRotationsConfig(), new MockCuratorDb(), new MetricsServiceMock()); + this(clock, defaultRotationsConfig(), new MockCuratorDb()); } public ControllerTester(RotationsConfig rotationsConfig) { - this(new ManualClock(), rotationsConfig, new MockCuratorDb(), new MetricsServiceMock()); + this(new ManualClock(), rotationsConfig, new MockCuratorDb()); } public ControllerTester(MockCuratorDb curatorDb) { - this(new ManualClock(), defaultRotationsConfig(), curatorDb, new MetricsServiceMock()); + this(new ManualClock(), defaultRotationsConfig(), curatorDb); } public ControllerTester() { @@ -108,8 +101,6 @@ public final class ControllerTester { private ControllerTester(AthenzDbMock athenzDb, ManualClock clock, ZoneRegistryMock zoneRegistry, CuratorDb curator, RotationsConfig rotationsConfig, - MockBuildService buildService, - MetricsServiceMock metricsService, ServiceRegistryMock serviceRegistry) { this.athenzDb = athenzDb; this.clock = clock; @@ -117,11 +108,7 @@ public final class ControllerTester { this.serviceRegistry = serviceRegistry; this.curator = curator; this.rotationsConfig = rotationsConfig; - this.buildService = buildService; - this.metricsService = metricsService; - this.controller = createController(curator, rotationsConfig, clock, zoneRegistry, - athenzDb, - metricsService, serviceRegistry); + this.controller = createController(curator, rotationsConfig, clock, zoneRegistry, athenzDb, serviceRegistry); // Make root logger use time from manual clock configureDefaultLogHandler(handler -> handler.setFilter( @@ -159,8 +146,6 @@ public final class ControllerTester { public ServiceRegistryMock serviceRegistry() { return serviceRegistry; } - public MetricsServiceMock metricsService() { return metricsService; } - public Optional<Record> findCname(String name) { return serviceRegistry.nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream().findFirst(); } @@ -168,7 +153,6 @@ public final class ControllerTester { /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(curator, rotationsConfig, clock, zoneRegistry, athenzDb, - metricsService, serviceRegistry); } @@ -305,12 +289,10 @@ public final class ControllerTester { ManualClock clock, ZoneRegistryMock zoneRegistryMock, AthenzDbMock athensDb, - MetricsServiceMock metricsService, ServiceRegistryMock serviceRegistry) { Controller controller = new Controller(curator, rotationsConfig, zoneRegistryMock, - metricsService, clock, new AthenzFacade(new AthenzClientFactoryMock(athensDb)), () -> "test-controller", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 4d16b546d6b..4a7ee8bcb63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -64,6 +64,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>(); private final Map<DeploymentId, List<Log>> warnings = new HashMap<>(); private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>(); + private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); private Version lastPrepareVersion = null; private RuntimeException prepareException = null; @@ -188,6 +189,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return Collections.unmodifiableMap(rotationNames); } + public void setMetrics(DeploymentId deployment, ClusterMetrics clusterMetrics) { + setMetrics(deployment, List.of(clusterMetrics)); + } + + public void setMetrics(DeploymentId deployment, List<ClusterMetrics> clusterMetrics) { + this.clusterMetrics.put(deployment, clusterMetrics); + } + @Override public NodeRepositoryMock nodeRepository() { return nodeRepository; @@ -336,7 +345,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public List<ClusterMetrics> getMetrics(DeploymentId deployment) { - return List.of(); + return Collections.unmodifiableList(clusterMetrics.getOrDefault(deployment, List.of())); } // Returns a canned example response diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java deleted file mode 100644 index 2eae50cbf9f..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.integration; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author bratseth - */ -public class MetricsServiceMock extends AbstractComponent implements MetricsService { - - private final Map<String, Double> metrics = new HashMap<>(); - - public MetricsServiceMock setMetric(String key, Double value) { - metrics.put(key, value); - return this; - } - - @Override - public ApplicationMetrics getApplicationMetrics(ApplicationId application) { - return new ApplicationMetrics(metrics.getOrDefault("queryServiceQuality", 0.5), - metrics.getOrDefault("writeServiceQuality", 0.7)); - } - - @Override - public DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone) { - return new DeploymentMetrics(metrics.getOrDefault("queriesPerSecond", 1D), - metrics.getOrDefault("writesPerSecond", 2D), - metrics.getOrDefault("docoumentCount", 3D).longValue(), - metrics.getOrDefault("queryLatencyMillis", 4D), - metrics.getOrDefault("writeLatencyMillis", 5D)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index 424cb05ebef..c8b3cd01fe8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -8,13 +8,17 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -28,14 +32,15 @@ import static org.junit.Assert.assertFalse; */ public class DeploymentMetricsMaintainerTest { - ControllerTester tester = new ControllerTester(); + private final DeploymentTester tester = new DeploymentTester(); @Test public void updates_metrics() { - ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1", - Environment.dev, 123).id(); + var application = tester.createApplication("app1", "tenant1", 123L, 1L); + deploy(application, Version.fromString("7.1")); + DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); - Supplier<Application> app = tester.application(appId); + Supplier<Application> app = () -> tester.application(application.id()); Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get(); // No metrics gathered yet @@ -48,7 +53,7 @@ public class DeploymentMetricsMaintainerTest { // Only get application metrics for old version deploy(app.get(), Version.fromString("6.3.3")); maintainer.maintain(); - assertEquals(0.5, app.get().metrics().queryServiceQuality(), 0); + assertEquals(0, app.get().metrics().queryServiceQuality(), 0); assertEquals(0, deployment.get().metrics().documentCount(), 0); assertFalse("No timestamp set", deployment.get().metrics().instant().isPresent()); assertFalse("Never received any queries", deployment.get().activity().lastQueried().isPresent()); @@ -56,10 +61,16 @@ public class DeploymentMetricsMaintainerTest { // Metrics are gathered and saved to application deploy(app.get(), Version.fromString("7.5.5")); + var metrics0 = Map.of(ClusterMetrics.QUERIES_PER_SECOND, 1D, + ClusterMetrics.FEED_PER_SECOND, 2D, + ClusterMetrics.DOCUMENT_COUNT, 3D, + ClusterMetrics.QUERY_LATENCY, 4D, + ClusterMetrics.FEED_LATENCY, 5D); + setMetrics(application.id(), metrics0); maintainer.maintain(); Instant t1 = tester.clock().instant().truncatedTo(MILLIS); - assertEquals(0.5, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE); - assertEquals(0.7, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE); + assertEquals(0.0, app.get().metrics().queryServiceQuality(), Double.MIN_VALUE); + assertEquals(0.0, app.get().metrics().writeServiceQuality(), Double.MIN_VALUE); assertEquals(1, deployment.get().metrics().queriesPerSecond(), Double.MIN_VALUE); assertEquals(2, deployment.get().metrics().writesPerSecond(), Double.MIN_VALUE); assertEquals(3, deployment.get().metrics().documentCount(), Double.MIN_VALUE); @@ -82,8 +93,10 @@ public class DeploymentMetricsMaintainerTest { // Query traffic disappears. Query activity stops updating tester.clock().advance(Duration.ofHours(1)); Instant t3 = tester.clock().instant().truncatedTo(MILLIS); - tester.metricsService().setMetric("queriesPerSecond", 0D); - tester.metricsService().setMetric("writesPerSecond", 5D); + var metrics1 = new HashMap<>(metrics0); + metrics1.put(ClusterMetrics.QUERIES_PER_SECOND, 0D); + metrics1.put(ClusterMetrics.FEED_PER_SECOND, 5D); + setMetrics(application.id(), metrics1); maintainer.maintain(); assertEquals(t2, deployment.get().activity().lastQueried().get()); assertEquals(t3, deployment.get().activity().lastWritten().get()); @@ -92,7 +105,9 @@ public class DeploymentMetricsMaintainerTest { // Feed traffic disappears. Feed activity stops updating tester.clock().advance(Duration.ofHours(1)); - tester.metricsService().setMetric("writesPerSecond", 0D); + var metrics2 = new HashMap<>(metrics1); + metrics2.put(ClusterMetrics.FEED_PER_SECOND, 0D); + setMetrics(application.id(), metrics2); maintainer.maintain(); assertEquals(t2, deployment.get().activity().lastQueried().get()); assertEquals(t3, deployment.get().activity().lastWritten().get()); @@ -100,15 +115,24 @@ public class DeploymentMetricsMaintainerTest { assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); } + private void setMetrics(ApplicationId application, Map<String, Double> metrics) { + var clusterMetrics = new ClusterMetrics("default", ClusterMetrics.ClusterType.container); + for (var kv : metrics.entrySet()) { + clusterMetrics = clusterMetrics.addMetric(kv.getKey(), kv.getValue()); + } + tester.controllerTester().serviceRegistry().configServerMock().setMetrics(new DeploymentId(application, ZoneId.from("dev", "us-east-1")), clusterMetrics); + } + private static DeploymentMetricsMaintainer maintainer(Controller controller) { return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator())); } private void deploy(Application application, Version version) { - tester.deploy(application, - ZoneId.from(Environment.dev, RegionName.from("us-east-1")), - Optional.of(new ApplicationPackage(new byte[0])), - false, - Optional.of(version)); + tester.controllerTester().deploy(application, + ZoneId.from(Environment.dev, RegionName.from("us-east-1")), + Optional.of(new ApplicationPackage(new byte[0])), + false, + Optional.of(version)); } + } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/ConfigServerMetricsServiceTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java index 7a271132e7d..23a6c6286c0 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/metrics/ConfigServerMetricsServiceTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/metric/ConfigServerMetricsTest.java @@ -1,19 +1,22 @@ -package com.yahoo.vespa.hosted.controller.api.integration.metrics; +package com.yahoo.vespa.hosted.controller.metric; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import java.util.List; import static org.junit.Assert.assertEquals; -public class ConfigServerMetricsServiceTest { +/** + * @author olaa + */ +public class ConfigServerMetricsTest { private final ApplicationId applicationId = new ApplicationId.Builder() .tenant("foo") @@ -23,13 +26,13 @@ public class ConfigServerMetricsServiceTest { private final ZoneId zoneId = ZoneId.from("prod", "us-west-1"); - private ConfigServer configServer; - private ConfigServerMetricsService service; + private ConfigServerMock configServer; + private ConfigServerMetrics service; @Before public void before() { - configServer = Mockito.mock(ConfigServer.class); - service = new ConfigServerMetricsService(configServer); + configServer = new ConfigServerMock(new ZoneRegistryMock()); + service = new ConfigServerMetrics(configServer); } @Test @@ -51,7 +54,7 @@ public class ConfigServerMetricsServiceTest { var response = List.of(clusterMetrics1, clusterMetrics2); - Mockito.when(configServer.getMetrics(deploymentId)).thenReturn(response); + configServer.setMetrics(deploymentId, response); // // Now we can actually test stuff :( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index d0e18be270a..16f57a61916 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; @@ -25,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationState; import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; @@ -114,7 +114,7 @@ public class ApplicationSerializerTest { Optional.of(IssueId.from("1234")), Optional.of(User.from("by-username")), OptionalInt.of(7), - new MetricsService.ApplicationMetrics(0.5, 0.9), + new ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----"), List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of())), rotationStatus); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 0fc7bcfc76f..0cfb79a0743 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -68,7 +68,6 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.Controller'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock'/>\n" + - " <component id='com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository'/>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index ec6c905b7e7..c6cd388d75f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -31,7 +31,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -60,6 +59,7 @@ import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater; +import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; |