Cache rotation status in DeploymentMetricsMaintainer

9 files changed, 239 insertions, 63 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index 677f2363c08..767b5e8328c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller;
 
 import com.google.common.collect.ImmutableMap;
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
@@ -16,12 +17,12 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 
 import java.time.Instant;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -49,47 +50,41 @@ public class Application {
     private final Optional<IssueId> ownershipIssueId;
     private final ApplicationMetrics metrics;
     private final Optional<RotationId> rotation;
+    private final Map<HostName, RotationStatus> rotationStatus;
 
     /** Creates an empty application */
     public Application(ApplicationId id) {
         this(id, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(),
              new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false),
              Change.empty(), Change.empty(), Optional.empty(), new ApplicationMetrics(0, 0),
-             Optional.empty());
+             Optional.empty(), Collections.emptyMap());
     }
 
     /** Used from persistence layer: Do not use */
     public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                        List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                        Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics,
-                       Optional<RotationId> rotation) {
+                       Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) {
         this(id, deploymentSpec, validationOverrides,
              deployments.stream().collect(Collectors.toMap(Deployment::zone, d -> d)),
-             deploymentJobs, change, outstandingChange, ownershipIssueId, metrics, rotation);
+             deploymentJobs, change, outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                 Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                 Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics,
-                Optional<RotationId> rotation) {
-        Objects.requireNonNull(id, "id cannot be null");
-        Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
-        Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null");
-        Objects.requireNonNull(deployments, "deployments cannot be null");
-        Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null");
-        Objects.requireNonNull(change, "change cannot be null");
-        Objects.requireNonNull(metrics, "metrics cannot be null");
-        Objects.requireNonNull(rotation, "rotation cannot be null");
-        this.id = id;
-        this.deploymentSpec = deploymentSpec;
-        this.validationOverrides = validationOverrides;
-        this.deployments = ImmutableMap.copyOf(deployments);
-        this.deploymentJobs = deploymentJobs;
-        this.change = change;
-        this.outstandingChange = outstandingChange;
-        this.ownershipIssueId = ownershipIssueId;
-        this.metrics = metrics;
-        this.rotation = rotation;
+                Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) {
+        this.id = Objects.requireNonNull(id, "id cannot be null");
+        this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
+        this.validationOverrides = Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null");
+        this.deployments = ImmutableMap.copyOf(Objects.requireNonNull(deployments, "deployments cannot be null"));
+        this.deploymentJobs = Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null");
+        this.change = Objects.requireNonNull(change, "change cannot be null");
+        this.outstandingChange = Objects.requireNonNull(outstandingChange, "outstandingChange cannot be null");
+        this.ownershipIssueId = Objects.requireNonNull(ownershipIssueId, "ownershipIssueId cannot be null");
+        this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
+        this.rotation = Objects.requireNonNull(rotation, "rotation cannot be null");
+        this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"));
     }
 
     public ApplicationId id() { return id; }
@@ -182,6 +177,23 @@ public class Application {
         return rotation.map(rotation -> new ApplicationRotation(id, rotation));
     }
 
+    /** Returns the status of the global rotation assigned to this. Wil be empty if this does not have a global rotation. */
+    public Map<HostName, RotationStatus> rotationStatus() {
+        return rotationStatus;
+    }
+
+    /** Returns the global rotation status of given deployment */
+    public RotationStatus rotationStatus(Deployment deployment) {
+        // Rotation status only contains VIP host names, one per zone in the system. The only way to map VIP hostname to
+        // this deployment, and thereby determine rotation status, is to check if VIP hostname contains the
+        // deployment's environment and region.
+        return rotationStatus.entrySet().stream()
+                             .filter(kv -> kv.getKey().value().contains(deployment.zone().value()))
+                             .map(Map.Entry::getValue)
+                             .findFirst()
+                             .orElse(RotationStatus.unknown);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 1576ab597be..9f4b58f69b5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -168,6 +168,7 @@ public class Controller extends AbstractComponent {
 
     public ZoneRegistry zoneRegistry() { return zoneRegistry; }
 
+    // TODO: Remove once DeploymentMetricsMaintainer has stored rotation status for all applications at least once
     public Map<String, RotationStatus> rotationStatus(Rotation rotation) {
         return globalRoutingService.getHealthStatus(rotation.name());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 2209cdf3013..e8d50d72669 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -6,10 +6,12 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationRotation;
 import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
@@ -49,6 +51,7 @@ public class LockedApplication {
     private final Optional<IssueId> ownershipIssueId;
     private final ApplicationMetrics metrics;
     private final Optional<RotationId> rotation;
+    private final Map<HostName, RotationStatus> rotationStatus;
 
     /**
      * Used to create a locked application
@@ -62,14 +65,15 @@ public class LockedApplication {
              application.deployments(),
              application.deploymentJobs(), application.change(), application.outstandingChange(),
              application.ownershipIssueId(), application.metrics(),
-             application.rotation().map(ApplicationRotation::id));
+             application.rotation().map(ApplicationRotation::id),
+             application.rotationStatus());
     }
 
     private LockedApplication(Lock lock, ApplicationId id,
                               DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                               Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                               Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics,
-                              Optional<RotationId> rotation) {
+                              Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) {
         this.lock = lock;
         this.id = id;
         this.deploymentSpec = deploymentSpec;
@@ -81,43 +85,44 @@ public class LockedApplication {
         this.ownershipIssueId = ownershipIssueId;
         this.metrics = metrics;
         this.rotation = rotation;
+        this.rotationStatus = rotationStatus;
     }
 
     /** Returns a read-only copy of this */
     public Application get() {
         return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change,
-                               outstandingChange, ownershipIssueId, metrics, rotation);
+                               outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withBuiltInternally(boolean builtInternally) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withProjectId(OptionalLong projectId) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withProjectId(projectId), change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withDeploymentIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.with(issueId), change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion,
                                                Optional<DeploymentJobs.JobError> jobError) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withCompletion(projectId, jobType, completion, jobError),
-                                     change, outstandingChange, ownershipIssueId, metrics, rotation);
+                                     change, outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withTriggering(jobType, job), change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version,
@@ -167,49 +172,54 @@ public class LockedApplication {
     public LockedApplication withoutDeploymentJob(JobType jobType) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.without(jobType), change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication with(DeploymentSpec deploymentSpec) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication with(ValidationOverrides validationOverrides) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withChange(Change change) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withOutstandingChange(Change outstandingChange) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication withOwnershipIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     Optional.ofNullable(issueId), metrics, rotation);
+                                     Optional.ofNullable(issueId), metrics, rotation, rotationStatus);
     }
 
     public LockedApplication with(MetricsService.ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     public LockedApplication with(RotationId rotation) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, Optional.of(rotation));
+                                     ownershipIssueId, metrics, Optional.of(rotation), rotationStatus);
+    }
+
+    public LockedApplication withRotationStatus(Map<HostName, RotationStatus> rotationStatus) {
+        return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments, deploymentJobs, change,
+                                     outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     /** Don't expose non-leaf sub-objects. */
@@ -222,7 +232,7 @@ public class LockedApplication {
     private LockedApplication with(Map<ZoneId, Deployment> deployments) {
         return new LockedApplication(lock, id, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
-                                     ownershipIssueId, metrics, rotation);
+                                     ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RotationStatus.java
new file mode 100644
index 00000000000..c9e174e7191
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RotationStatus.java
@@ -0,0 +1,20 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+/**
+ * Represents the health status of a global rotation.
+ *
+ * @author mpolden
+ */
+public enum RotationStatus {
+
+    /** Rotation has status 'in' and is receiving traffic */
+    in,
+
+    /** Rotation has status 'out' and is *NOT* receiving traffic */
+    out,
+
+    /** Rotation status is currently unknown, or no global rotation has been assigned */
+    unknown
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 4dacb2e32d6..04a6e4075f7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -1,16 +1,22 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
 
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.UncheckedIOException;
 import java.time.Duration;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -24,17 +30,23 @@ public class DeploymentMetricsMaintainer extends Maintainer {
 
     private static final Logger log = Logger.getLogger(DeploymentMetricsMaintainer.class.getName());
 
+    private final ApplicationController applications;
+
     DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) {
         super(controller, duration, jobControl);
+        this.applications = controller.applications();
     }
 
     @Override
     protected void maintain() {
         boolean hasWarned = false;
-        for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) {
+        for (Application application : ApplicationList.from(applications.asList()).notPullRequest().asList()) {
             try {
-                controller().applications().lockIfPresent(application.id(), lockedApplication ->
-                        controller().applications().store(lockedApplication.with(controller().metricsService().getApplicationMetrics(application.id()))));
+                applications.lockIfPresent(application.id(), locked ->
+                        applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id()))));
+
+                applications.lockIfPresent(application.id(), locked ->
+                        applications.store(locked.withRotationStatus(rotationStatus(application))));
 
                 for (Deployment deployment : application.deployments().values()) {
                     MetricsService.DeploymentMetrics deploymentMetrics = controller().metricsService()
@@ -45,9 +57,9 @@ public class DeploymentMetricsMaintainer extends Maintainer {
                                                                          deploymentMetrics.queryLatencyMillis(),
                                                                          deploymentMetrics.writeLatencyMillis());
 
-                    controller().applications().lockIfPresent(application.id(), lockedApplication ->
-                            controller().applications().store(lockedApplication.with(deployment.zone(), newMetrics)
-                                                                               .recordActivityAt(controller().clock().instant(), deployment.zone())));
+                    applications.lockIfPresent(application.id(), locked ->
+                            applications.store(locked.with(deployment.zone(), newMetrics)
+                                                     .recordActivityAt(controller().clock().instant(), deployment.zone())));
                 }
             } catch (UncheckedIOException e) {
                 if (!hasWarned) // produce only one warning per maintenance interval
@@ -58,4 +70,25 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         }
     }
 
+    /** Get global rotation status for application */
+    private Map<HostName, RotationStatus> rotationStatus(Application application) {
+        return application.rotation()
+                          .map(rotation -> controller().metricsService().getRotationStatus(rotation.id().asString()))
+                          .map(rotationStatus -> {
+                              Map<HostName, RotationStatus> result = new TreeMap<>();
+                              rotationStatus.forEach((hostname, status) -> result.put(hostname, from(status)));
+                              return result;
+                          })
+                          .orElseGet(Collections::emptyMap);
+    }
+
+    private static RotationStatus from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) {
+        switch (status) {
+            case IN: return RotationStatus.in;
+            case OUT: return RotationStatus.out;
+            case UNKNOWN: return RotationStatus.unknown;
+            default: throw new IllegalArgumentException("Unknown API value for rotation status: " + status);
+        }
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 58e0b8dbeec..9ee23dbe51b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -26,18 +27,21 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalDouble;
 import java.util.OptionalLong;
+import java.util.TreeMap;
 
 /**
  * Serializes applications to/from slime.
@@ -59,6 +63,7 @@ public class ApplicationSerializer {
     private final String writeQualityField = "writeQuality";
     private final String queryQualityField = "queryQuality";
     private final String rotationField = "rotation";
+    private final String rotationStatusField = "rotationStatus";
 
     // Deployment fields
     private final String zoneField = "zone";
@@ -124,7 +129,6 @@ public class ApplicationSerializer {
     private final String deploymentMetricsQueryLatencyField = "queryLatencyMillis";
     private final String deploymentMetricsWriteLatencyField = "writeLatencyMillis";
 
-
     // ------------------ Serialization
 
     public Slime toSlime(Application application) {
@@ -141,6 +145,7 @@ public class ApplicationSerializer {
         root.setDouble(queryQualityField, application.metrics().queryServiceQuality());
         root.setDouble(writeQualityField, application.metrics().writeServiceQuality());
         application.rotation().ifPresent(rotation -> root.setString(rotationField, rotation.id().asString()));
+        toSlime(application.rotationStatus(), root.setArray(rotationStatusField));
         return slime;
     }
 
@@ -268,6 +273,14 @@ public class ApplicationSerializer {
             toSlime(deploying.application().get(), object);
     }
 
+    private void toSlime(Map<HostName, RotationStatus> rotationStatus, Cursor array) {
+        rotationStatus.forEach((hostname, status) -> {
+            Cursor object = array.addObject();
+            object.setString("hostname", hostname.value());
+            object.setString("status", status.name());
+        });
+    }
+
     // ------------------ Deserialization
 
     public Application fromSlime(Slime slime) {
@@ -284,9 +297,10 @@ public class ApplicationSerializer {
         ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
                                                             root.field(writeQualityField).asDouble());
         Optional<RotationId> rotation = rotationFromSlime(root.field(rotationField));
+        Map<HostName, RotationStatus> rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField));
 
         return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying,
-                               outstandingChange, ownershipIssueId, metrics, rotation);
+                               outstandingChange, ownershipIssueId, metrics, rotation, rotationStatus);
     }
 
     private List<Deployment> deploymentsFromSlime(Inspector array) {
@@ -317,6 +331,19 @@ public class ApplicationSerializer {
                                      object.field(deploymentMetricsWriteLatencyField).asDouble());
     }
 
+    private Map<HostName, RotationStatus> rotationStatusFromSlime(Inspector object) {
+        if (!object.valid()) {
+            return Collections.emptyMap();
+        }
+        Map<HostName, RotationStatus> rotationStatus = new TreeMap<>();
+        object.traverse((ArrayTraverser) (idx, inspect) -> {
+            HostName hostname = HostName.from(inspect.field("hostname").asString());
+            RotationStatus status = RotationStatus.valueOf(inspect.field("status").asString());
+            rotationStatus.put(hostname, status);
+        });
+        return Collections.unmodifiableMap(rotationStatus);
+    }
+
     private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime(Inspector object) {
         Map<ClusterSpec.Id, ClusterInfo> map = new HashMap<>();
         object.traverse((String name, Inspector obect) -> map.put(new ClusterSpec.Id(name), clusterInfoFromSlime(obect)));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java
index eca78c01e09..d3b5fdf9415 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsServiceMock.java
@@ -1,8 +1,10 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 
 import java.util.HashMap;
@@ -14,12 +16,23 @@ import java.util.Map;
 public class MetricsServiceMock implements MetricsService {
 
     private final Map<String, Double> metrics = new HashMap<>();
+    private final Map<HostName, RotationStatus> rotationStatus = new HashMap<>();
 
     public MetricsServiceMock setMetric(String key, Double value) {
         metrics.put(key, value);
         return this;
     }
 
+    public MetricsServiceMock setRotationIn(String hostname) {
+        rotationStatus.put(HostName.from(hostname), RotationStatus.IN);
+        return this;
+    }
+
+    public MetricsServiceMock setRotationOut(String hostname) {
+        rotationStatus.put(HostName.from(hostname), RotationStatus.OUT);
+        return this;
+    }
+
     @Override
     public ApplicationMetrics getApplicationMetrics(ApplicationId application) {
         return new ApplicationMetrics(metrics.getOrDefault("queryServiceQuality", 0.5),
@@ -43,4 +56,9 @@ public class MetricsServiceMock implements MetricsService {
         return result;
     }
 
+    @Override
+    public Map<HostName, RotationStatus> getRotationStatus(String rotationName) {
+        return rotationStatus;
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index d3e42bae526..91958cf838e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -1,14 +1,18 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -26,13 +30,11 @@ import static org.junit.Assert.assertFalse;
 public class DeploymentMetricsMaintainerTest {
 
     @Test
-    public void maintain() {
+    public void updates_metrics() {
         ControllerTester tester = new ControllerTester();
         ApplicationId appId = tester.createAndDeploy("tenant1", "domain1", "app1",
                                                   Environment.dev, 123).id();
-        DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(),
-                                                                                 Duration.ofDays(1),
-                                                                                 new JobControl(new MockCuratorDb()));
+        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
         Supplier<Application> app = tester.application(appId);
         Supplier<Deployment> deployment = () -> app.get().deployments().values().stream().findFirst().get();
 
@@ -85,4 +87,48 @@ public class DeploymentMetricsMaintainerTest {
         assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
     }
 
+    @Test
+    public void updates_rotation_status() {
+        DeploymentTester tester = new DeploymentTester();
+        MetricsServiceMock metricsService = tester.controllerTester().metricsService();
+        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
+        Application application = tester.createApplication("app1", "tenant1", 1, 1L);
+        ZoneId zone1 = ZoneId.from("prod", "us-west-1");
+        ZoneId zone2 = ZoneId.from("prod", "us-east-3");
+
+        // Deploy application with global rotation
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .globalServiceId("foo")
+                .region(zone1.region().value())
+                .region(zone2.region().value())
+                .build();
+        tester.deployCompletely(application, applicationPackage);
+
+        Supplier<Application> app = () -> tester.application(application.id());
+        Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1);
+        Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2);
+
+        // No status gathered yet
+        assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment1.get()));
+        assertEquals(RotationStatus.unknown, app.get().rotationStatus(deployment2.get()));
+
+        // One rotation out, one in
+        metricsService.setRotationIn("proxy.prod.us-west-1.vip.test");
+        metricsService.setRotationOut("proxy.prod.us-east-3.vip.test");
+        maintainer.maintain();
+        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
+        assertEquals(RotationStatus.out, app.get().rotationStatus(deployment2.get()));
+
+        // All rotations in
+        metricsService.setRotationIn("proxy.prod.us-east-3.vip.test");
+        maintainer.maintain();
+        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment1.get()));
+        assertEquals(RotationStatus.in, app.get().rotationStatus(deployment2.get()));
+    }
+
+    private static DeploymentMetricsMaintainer maintainer(Controller controller) {
+        return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index cf543bdd2c0..8d99dfc5c16 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService;
@@ -22,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import org.junit.Test;
@@ -37,6 +39,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalDouble;
 import java.util.OptionalLong;
+import java.util.TreeMap;
 
 import static com.yahoo.config.provision.SystemName.main;
 import static com.yahoo.vespa.hosted.controller.ControllerTester.writable;
@@ -89,6 +92,10 @@ public class ApplicationSerializerTest {
 
         DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty(), true);
 
+        Map<HostName, RotationStatus> rotationStatus = new TreeMap<>();
+        rotationStatus.put(HostName.from("rot1.fqdn"), RotationStatus.in);
+        rotationStatus.put(HostName.from("rot2.fqdn"), RotationStatus.out);
+
         Application original = new Application(ApplicationId.from("t1", "a1", "i1"),
                                                deploymentSpec,
                                                validationOverrides,
@@ -97,7 +104,8 @@ public class ApplicationSerializerTest {
                                                Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)),
                                                Optional.of(IssueId.from("1234")),
                                                new MetricsService.ApplicationMetrics(0.5, 0.9),
-                                               Optional.of(new RotationId("my-rotation")));
+                                               Optional.of(new RotationId("my-rotation")),
+                                               rotationStatus);
 
         Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original));
 
@@ -129,6 +137,7 @@ public class ApplicationSerializerTest {
 
         assertEquals(original.change(), serialized.change());
         assertEquals(original.rotation().get().id(), serialized.rotation().get().id());
+        assertEquals(original.rotationStatus(), serialized.rotationStatus());
 
         // Test cluster utilization
         assertEquals(0, serialized.deployments().get(zone1).clusterUtils().size());