diff options
author | Martin Polden <mpolden@mpolden.no> | 2018-09-21 09:44:26 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-09-21 09:44:26 +0200 |
commit | 0fbc4d579dd3a05a42dc224f7e883d56051052f4 (patch) | |
tree | 41afd9b03843fd6efeac9b9b0d178d6f29cdce89 | |
parent | f53da8ecfc3a2aa499b75a7599829c0d1ce73c50 (diff) | |
parent | 8f2d713dca812b9608f6ca2be5d877653b3a2a86 (diff) |
Merge pull request #7031 from vespa-engine/mpolden/cache-rotation-status
Cache rotation status in DeploymentMetricsMaintainer
14 files changed, 292 insertions, 93 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java index 20e9710f092..ff0628ccccc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MetricsService.java @@ -1,9 +1,12 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import java.util.Collections; import java.util.Map; /** @@ -17,6 +20,11 @@ public interface MetricsService { DeploymentMetrics getDeploymentMetrics(ApplicationId application, ZoneId zone); + // TODO: Remove default once implementation catches up + default Map<HostName, RotationStatus> getRotationStatus(String rotationName) { + return Collections.emptyMap(); + } + Map<String, SystemMetrics> getSystemMetrics(ApplicationId application, ZoneId zone); class DeploymentMetrics { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java index d49d6a9e4c2..5a5f775c013 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java @@ -8,6 +8,7 @@ import java.util.Map; * * @author mpolden */ +// TODO: Remove once DeploymentMetricsMaintainer starts providing rotation status public interface GlobalRoutingService { /** Returns the health status for each endpoint behind the given rotation name */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 677f2363c08..767b5e8328c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableMap; @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -16,12 +17,12 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.Collections; import java.util.Comparator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -49,47 +50,41 @@ public class Application { private final Optional<IssueId> ownershipIssueId; private final ApplicationMetrics metrics; private final Optional<RotationId> rotation; + private final Map<HostName, RotationStatus> rotationStatus; /** Creates an empty application */ public Application(ApplicationId id) { this(id, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(), new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), new ApplicationMetrics(0, 0), - Optional.empty()); + Optional.empty(), Collections.emptyMap()); } /** Used from persistence layer: Do not use */ public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, - Optional<RotationId> rotation) { + Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) { this(id, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, d -> d)), - deploymentJobs, change, outstandingChange, ownershipIssueId, metrics, rotation); + deploymentJobs, change, outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus); } Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, - Optional<RotationId> rotation) { - Objects.requireNonNull(id, "id cannot be null"); - Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); - Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null"); - Objects.requireNonNull(deployments, "deployments cannot be null"); - Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null"); - Objects.requireNonNull(change, "change cannot be null"); - Objects.requireNonNull(metrics, "metrics cannot be null"); - Objects.requireNonNull(rotation, "rotation cannot be null"); - this.id = id; - this.deploymentSpec = deploymentSpec; - this.validationOverrides = validationOverrides; - this.deployments = ImmutableMap.copyOf(deployments); - this.deploymentJobs = deploymentJobs; - this.change = change; - this.outstandingChange = outstandingChange; - this.ownershipIssueId = ownershipIssueId; - this.metrics = metrics; - this.rotation = rotation; + Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) { + this.id = Objects.requireNonNull(id, "id cannot be null"); + this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); + this.validationOverrides = Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null"); + this.deployments = ImmutableMap.copyOf(Objects.requireNonNull(deployments, "deployments cannot be null")); + this.deploymentJobs = Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null"); + this.change = Objects.requireNonNull(change, "change cannot be null"); + this.outstandingChange = Objects.requireNonNull(outstandingChange, "outstandingChange cannot be null"); + this.ownershipIssueId = Objects.requireNonNull(ownershipIssueId, "ownershipIssueId cannot be null"); + this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); + this.rotation = Objects.requireNonNull(rotation, "rotation cannot be null"); + this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); } public ApplicationId id() { return id; } @@ -182,6 +177,23 @@ public class Application { return rotation.map(rotation -> new ApplicationRotation(id, rotation)); } + /** Returns the status of the global rotation assigned to this. Wil be empty if this does not have a global rotation. */ + public Map<HostName, RotationStatus> rotationStatus() { + return rotationStatus; + } + + /** Returns the global rotation status of given deployment */ + public RotationStatus rotationStatus(Deployment deployment) { + // Rotation status only contains VIP host names, one per zone in the system. The only way to map VIP hostname to + // this deployment, and thereby determine rotation status, is to check if VIP hostname contains the + // deployment's environment and region. + return rotationStatus.entrySet().stream() + .filter(kv -> kv.getKey().value().contains(deployment.zone().value())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(RotationStatus.unknown); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 1576ab597be..9f4b58f69b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -168,6 +168,7 @@ public class Controller extends AbstractComponent { public ZoneRegistry zoneRegistry() { return zoneRegistry; } + // TODO: Remove once DeploymentMetricsMaintainer has stored rotation status for all applications at least once public Map<String, RotationStatus> rotationStatus(Rotation rotation) { return globalRoutingService.getHealthStatus(rotation.name()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 2209cdf3013..e8d50d72669 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -6,10 +6,12 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; @@ -49,6 +51,7 @@ public class LockedApplication { private final Optional<IssueId> ownershipIssueId; private final ApplicationMetrics metrics; private final Optional<RotationId> rotation; + private final Map<HostName, RotationStatus> rotationStatus; /** * Used to create a locked application @@ -62,14 +65,15 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.metrics(), - application.rotation().map(ApplicationRotation::id)); + application.rotation().map(ApplicationRotation::id), + application.rotationStatus()); } private LockedApplication(Lock lock, ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, - Optional<RotationId> rotation) { + Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) { this.lock = lock; this.id = id; this.deploymentSpec = deploymentSpec; @@ -81,43 +85,44 @@ public class LockedApplication { this.ownershipIssueId = ownershipIssueId; this.metrics = metrics; this.rotation = rotation; + this.rotationStatus = rotationStatus; } /** Returns a read-only copy of this */ public Application get() { return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, - outstandingChange, ownershipIssueId, metrics, rotation); + outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.with(issueId), change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, Optional<DeploymentJobs.JobError> jobError) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withCompletion(projectId, jobType, completion, jobError), - change, outstandingChange, ownershipIssueId, metrics, rotation); + change, outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.withTriggering(jobType, job), change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -167,49 +172,54 @@ public class LockedApplication { public LockedApplication withoutDeploymentJob(JobType jobType) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - Optional.ofNullable(issueId), metrics, rotation); + Optional.ofNullable(issueId), metrics, rotation, rotationStatus); } public LockedApplication with(MetricsService.ApplicationMetrics metrics) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } public LockedApplication with(RotationId rotation) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, Optional.of(rotation)); + ownershipIssueId, metrics, Optional.of(rotation), rotationStatus); + } + + public LockedApplication withRotationStatus(Map<HostName, RotationStatus> rotationStatus) { + return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, + outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus); } /** Don't expose non-leaf sub-objects. */ @@ -222,7 +232,7 @@ public class LockedApplication { private LockedApplication with(Map<ZoneId, Deployment> deployments) { return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, - ownershipIssueId, metrics, rotation); + ownershipIssueId, metrics, rotation, rotationStatus); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index a2433d223dc..c099e856d04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -30,7 +30,7 @@ public class Deployment { public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) { this(zone, applicationVersion, version, deployTime, Collections.emptyMap(), Collections.emptyMap(), - new DeploymentMetrics(), DeploymentActivity.none); + DeploymentMetrics.none, DeploymentActivity.none); } public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java index c0f7bd6c6a1..35bc86cac6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java @@ -2,24 +2,20 @@ package com.yahoo.vespa.hosted.controller.application; /** + * Metrics for a deployment of an application. + * * @author smorgrav */ public class DeploymentMetrics { + public static final DeploymentMetrics none = new DeploymentMetrics(0, 0, 0, 0, 0); + private final double queriesPerSecond; private final double writesPerSecond; private final double documentCount; private final double queryLatencyMillis; private final double writeLatencyMills; - DeploymentMetrics() { - this.queriesPerSecond = 0; - this.writesPerSecond = 0; - this.documentCount = 0; - this.queryLatencyMillis = 0; - this.writeLatencyMills = 0; - } - public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, double documentCount, double queryLatencyMillis, double writeLatencyMills) { this.queriesPerSecond = queriesPerSecond; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RotationStatus.java new file mode 100644 index 00000000000..c9e174e7191 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RotationStatus.java @@ -0,0 +1,20 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +/** + * Represents the health status of a global rotation. + * + * @author mpolden + */ +public enum RotationStatus { + + /** Rotation has status 'in' and is receiving traffic */ + in, + + /** Rotation has status 'out' and is *NOT* receiving traffic */ + out, + + /** Rotation status is currently unknown, or no global rotation has been assigned */ + unknown + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 4dacb2e32d6..73bc29ed47e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -1,16 +1,26 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.yolean.Exceptions; import java.io.UncheckedIOException; import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; @@ -24,37 +34,78 @@ public class DeploymentMetricsMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(DeploymentMetricsMaintainer.class.getName()); + private static final int applicationsToUpdateInParallel = 10; + + private final ApplicationController applications; + DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { super(controller, duration, jobControl); + this.applications = controller.applications(); } @Override protected void maintain() { - boolean hasWarned = false; - for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { - try { - controller().applications().lockIfPresent(application.id(), lockedApplication -> - controller().applications().store(lockedApplication.with(controller().metricsService().getApplicationMetrics(application.id())))); - - for (Deployment deployment : application.deployments().values()) { - MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() - .getDeploymentMetrics(application.id(), deployment.zone()); - DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(), - deploymentMetrics.writesPerSecond(), - deploymentMetrics.documentCount(), - deploymentMetrics.queryLatencyMillis(), - deploymentMetrics.writeLatencyMillis()); - - controller().applications().lockIfPresent(application.id(), lockedApplication -> - controller().applications().store(lockedApplication.with(deployment.zone(), newMetrics) - .recordActivityAt(controller().clock().instant(), deployment.zone()))); + AtomicBoolean hasWarned = new AtomicBoolean(false); + List<Application> applicationList = ApplicationList.from(applications.asList()).notPullRequest().asList(); + + // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used + ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel); + pool.submit(() -> { + applicationList.parallelStream().forEach(application -> { + try { + applications.lockIfPresent(application.id(), locked -> + applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id())))); + + applications.lockIfPresent(application.id(), locked -> + applications.store(locked.withRotationStatus(rotationStatus(application)))); + + for (Deployment deployment : application.deployments().values()) { + MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService() + .getDeploymentMetrics(application.id(), deployment.zone()); + DeploymentMetrics newMetrics = new DeploymentMetrics(deploymentMetrics.queriesPerSecond(), + deploymentMetrics.writesPerSecond(), + deploymentMetrics.documentCount(), + deploymentMetrics.queryLatencyMillis(), + deploymentMetrics.writeLatencyMillis()); + + applications.lockIfPresent(application.id(), locked -> + applications.store(locked.with(deployment.zone(), newMetrics) + .recordActivityAt(controller().clock().instant(), deployment.zone()))); + } + } catch (UncheckedIOException e) { + if (!hasWarned.getAndSet(true)) {// produce only one warning per maintenance interval + log.log(Level.WARNING, "Failed to query metrics service: " + Exceptions.toMessageString(e) + + ". Retrying in " + maintenanceInterval()); + } } - } catch (UncheckedIOException e) { - if (!hasWarned) // produce only one warning per maintenance interval - log.log(Level.WARNING, "Failed to query metrics service: " + Exceptions.toMessageString(e) + - ". Retrying in " + maintenanceInterval()); - hasWarned = true; - } + }); + }); + pool.shutdown(); + try { + pool.awaitTermination(30, TimeUnit.MINUTES); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** Get global rotation status for application */ + private Map<HostName, RotationStatus> rotationStatus(Application application) { + return application.rotation() + .map(rotation -> controller().metricsService().getRotationStatus(rotation.id().asString())) + .map(rotationStatus -> { + Map<HostName, RotationStatus> result = new TreeMap<>(); + rotationStatus.forEach((hostname, status) -> result.put(hostname, from(status))); + return result; + }) + .orElseGet(Collections::emptyMap); + } + + private static RotationStatus from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) { + switch (status) { + case IN: return RotationStatus.in; + case OUT: return RotationStatus.out; + case UNKNOWN: return RotationStatus.unknown; + default: throw new IllegalArgumentException("Unknown API value for rotation status: " + status); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 58e0b8dbeec..9ee23dbe51b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -26,18 +27,21 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalLong; +import java.util.TreeMap; /** * Serializes applications to/from slime. @@ -59,6 +63,7 @@ public class ApplicationSerializer { private final String writeQualityField = "writeQuality"; private final String queryQualityField = "queryQuality"; private final String rotationField = "rotation"; + private final String rotationStatusField = "rotationStatus"; // Deployment fields private final String zoneField = "zone"; @@ -124,7 +129,6 @@ public class ApplicationSerializer { private final String deploymentMetricsQueryLatencyField = "queryLatencyMillis"; private final String deploymentMetricsWriteLatencyField = "writeLatencyMillis"; - // ------------------ Serialization public Slime toSlime(Application application) { @@ -141,6 +145,7 @@ public class ApplicationSerializer { root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); application.rotation().ifPresent(rotation -> root.setString(rotationField, rotation.id().asString())); + toSlime(application.rotationStatus(), root.setArray(rotationStatusField)); return slime; } @@ -268,6 +273,14 @@ public class ApplicationSerializer { toSlime(deploying.application().get(), object); } + private void toSlime(Map<HostName, RotationStatus> rotationStatus, Cursor array) { + rotationStatus.forEach((hostname, status) -> { + Cursor object = array.addObject(); + object.setString("hostname", hostname.value()); + object.setString("status", status.name()); + }); + } + // ------------------ Deserialization public Application fromSlime(Slime slime) { @@ -284,9 +297,10 @@ public class ApplicationSerializer { ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); Optional<RotationId> rotation = rotationFromSlime(root.field(rotationField)); + Map<HostName, RotationStatus> rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField)); return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, - outstandingChange, ownershipIssueId, metrics, rotation); + outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus); } private List<Deployment> deploymentsFromSlime(Inspector array) { @@ -317,6 +331,19 @@ public class ApplicationSerializer { object.field(deploymentMetricsWriteLatencyField).asDouble()); } + private Map<HostName, RotationStatus> rotationStatusFromSlime(Inspector object) { + if (!object.valid()) { + return Collections.emptyMap(); + } + Map<HostName, RotationStatus> rotationStatus = new TreeMap<>(); + object.traverse((ArrayTraverser) (idx, inspect) -> { + HostName hostname = HostName.from(inspect.field("hostname").asString()); + RotationStatus status = RotationStatus.valueOf(inspect.field("status").asString()); + rotationStatus.put(hostname, status); + }); + return Collections.unmodifiableMap(rotationStatus); + } + private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime(Inspector object) { Map<ClusterSpec.Id, ClusterInfo> map = new HashMap<>(); object.traverse((String name, Inspector obect) -> map.put(new ClusterSpec.Id(name), clusterInfoFromSlime(obect))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java index eca78c01e09..d3b5fdf9415 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java @@ -1,8 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import java.util.HashMap; @@ -14,12 +16,23 @@ import java.util.Map; public class MetricsServiceMock implements MetricsService { private final Map<String, Double> metrics = new HashMap<>(); + private final Map<HostName, RotationStatus> rotationStatus = new HashMap<>(); public MetricsServiceMock setMetric(String key, Double value) { metrics.put(key, value); return this; } + public MetricsServiceMock setRotationIn(String hostname) { + rotationStatus.put(HostName.from(hostname), RotationStatus.IN); + return this; + } + + public MetricsServiceMock setRotationOut(String hostname) { + rotationStatus.put(HostName.from(hostname), RotationStatus.OUT); + return this; + } + @Override public ApplicationMetrics getApplicationMetrics(ApplicationId application) { return new ApplicationMetrics(metrics.getOrDefault("queryServiceQuality", 0.5), @@ -43,4 +56,9 @@ public class MetricsServiceMock implements MetricsService { return result; } + @Override + public Map<HostName, RotationStatus> getRotationStatus(String rotationName) { + return rotationStatus; + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index d3e42bae526..91958cf838e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -1,14 +1,18 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; import org.junit.Test; import java.time.Duration; @@ -26,13 +30,11 @@ import static org.junit.Assert.assertFalse; public class DeploymentMetricsMaintainerTest { @Test - public void maintain() { + public void updates_metrics() { ControllerTester tester = new ControllerTester(); ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); - DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), - Duration.ofDays(1), - new JobControl(new MockCuratorDb())); + DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); Supplier<Application> app = tester.application(appId); Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get(); @@ -85,4 +87,48 @@ public class DeploymentMetricsMaintainerTest { assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE); } + @Test + public void updates_rotation_status() { + DeploymentTester tester = new DeploymentTester(); + MetricsServiceMock metricsService = tester.controllerTester().metricsService(); + DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + ZoneId zone1 = ZoneId.from("prod", "us-west-1"); + ZoneId zone2 = ZoneId.from("prod", "us-east-3"); + + // Deploy application with global rotation + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .globalServiceId("foo") + .region(zone1.region().value()) + .region(zone2.region().value()) + .build(); + tester.deployCompletely(application, applicationPackage); + + Supplier<Application> app = () -> tester.application(application.id()); + Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1); + Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2); + + // No status gathered yet + assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get())); + assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get())); + + // One rotation out, one in + metricsService.setRotationIn("proxy.prod.us-west-1.vip.test"); + metricsService.setRotationOut("proxy.prod.us-east-3.vip.test"); + maintainer.maintain(); + assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get())); + assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get())); + + // All rotations in + metricsService.setRotationIn("proxy.prod.us-east-3.vip.test"); + maintainer.maintain(); + assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get())); + assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get())); + } + + private static DeploymentMetricsMaintainer maintainer(Controller controller) { + return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator())); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 40d245db8f0..8d99dfc5c16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; @@ -22,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import org.junit.Test; @@ -37,6 +39,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalLong; +import java.util.TreeMap; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.ControllerTester.writable; @@ -70,7 +73,7 @@ public class ApplicationSerializerTest { deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), createClusterUtils(3, 0.2), createClusterInfo(3, 4), - new DeploymentMetrics(2,3,4,5,6), + new DeploymentMetrics(2, 3, 4, 5, 6), DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt), OptionalDouble.of(200), OptionalDouble.of(10)))); @@ -89,6 +92,10 @@ public class ApplicationSerializerTest { DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty(), true); + Map<HostName, RotationStatus> rotationStatus = new TreeMap<>(); + rotationStatus.put(HostName.from("rot1.fqdn"), RotationStatus.in); + rotationStatus.put(HostName.from("rot2.fqdn"), RotationStatus.out); + Application original = new Application(ApplicationId.from("t1", "a1", "i1"), deploymentSpec, validationOverrides, @@ -97,7 +104,8 @@ public class ApplicationSerializerTest { Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)), Optional.of(IssueId.from("1234")), new MetricsService.ApplicationMetrics(0.5, 0.9), - Optional.of(new RotationId("my-rotation"))); + Optional.of(new RotationId("my-rotation")), + rotationStatus); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -129,6 +137,7 @@ public class ApplicationSerializerTest { assertEquals(original.change(), serialized.change()); assertEquals(original.rotation().get().id(), serialized.rotation().get().id()); + assertEquals(original.rotationStatus(), serialized.rotationStatus()); // Test cluster utilization assertEquals(0, serialized.deployments().get(zone1).clusterUtils().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 30c81a0721a..aa219aab05d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1198,7 +1198,7 @@ public class ApplicationApiTest extends ControllerContainerTest { clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); Map<ClusterSpec.Id, ClusterUtilization> clusterUtils = new HashMap<>(); clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); - DeploymentMetrics metrics = new DeploymentMetrics(1,2,3,4,5); + DeploymentMetrics metrics = new DeploymentMetrics(1, 2, 3, 4, 5); lockedApplication = lockedApplication .withClusterInfo(deployment.zone(), clusterInfo) |