Support storing status of multiple rotations per application

13 files changed, 251 insertions, 80 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index 881a2afb27a..48dc343ca8f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -7,7 +7,6 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
@@ -22,7 +21,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Instant;
 import java.util.Collections;
@@ -59,7 +58,7 @@ public class Application {
     private final ApplicationMetrics metrics;
     private final Optional<String> pemDeployKey;
     private final List<AssignedRotation> rotations;
-    private final Map<HostName, RotationState> rotationStatus;
+    private final RotationStatus rotationStatus;
     private final Optional<ApplicationCertificate> applicationCertificate;
 
     /** Creates an empty application */
@@ -68,7 +67,7 @@ public class Application {
              new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false),
              Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
              new ApplicationMetrics(0, 0),
-             Optional.empty(), Collections.emptyList(), Collections.emptyMap(), Optional.empty());
+             Optional.empty(), Collections.emptyList(), RotationStatus.EMPTY, Optional.empty());
     }
 
     /** Used from persistence layer: Do not use */
@@ -76,7 +75,7 @@ public class Application {
                        List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                        Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                        OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
-                       List<AssignedRotation> rotations, Map<HostName, RotationState> rotationStatus,
+                       List<AssignedRotation> rotations, RotationStatus rotationStatus,
                        Optional<ApplicationCertificate> applicationCertificate) {
         this(id, createdAt, deploymentSpec, validationOverrides,
              deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())),
@@ -88,7 +87,7 @@ public class Application {
                 Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                 Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                 OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
-                List<AssignedRotation> rotations, Map<HostName, RotationState> rotationStatus, Optional<ApplicationCertificate> applicationCertificate) {
+                List<AssignedRotation> rotations, RotationStatus rotationStatus, Optional<ApplicationCertificate> applicationCertificate) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
         this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
@@ -103,7 +102,7 @@ public class Application {
         this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
         this.pemDeployKey = pemDeployKey;
         this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null"));
-        this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"));
+        this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null");
         this.applicationCertificate = Objects.requireNonNull(applicationCertificate, "applicationCertificate cannot be null");
     }
 
@@ -220,23 +219,11 @@ public class Application {
 
     public Optional<String> pemDeployKey() { return pemDeployKey; }
 
-    /** Returns the status of the global rotation assigned to this. Empty if this does not have a global rotation. */
-    public Map<HostName, RotationState> rotationStatus() {
+    /** Returns the status of the global rotation(s) assigned to this */
+    public RotationStatus rotationStatus() {
         return rotationStatus;
     }
 
-    /** Returns the global rotation status of given deployment */
-    public RotationState rotationStatus(Deployment deployment) {
-        // Rotation status only contains VIP host names, one per zone in the system. The only way to map VIP hostname to
-        // this deployment, and thereby determine rotation status, is to check if VIP hostname contains the
-        // deployment's environment and region.
-        return rotationStatus.entrySet().stream()
-                             .filter(kv -> kv.getKey().value().contains(deployment.zone().value()))
-                             .map(Map.Entry::getValue)
-                             .findFirst()
-                             .orElse(RotationState.unknown);
-    }
-
     public Optional<ApplicationCertificate> applicationCertificate() {
         return applicationCertificate;
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index f642ee61ec4..7b0581b8ca9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -12,7 +12,6 @@ import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
@@ -20,8 +19,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.api.integration.user.Roles;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
@@ -85,6 +86,7 @@ public class Controller extends AbstractComponent {
     private final ApplicationCertificateProvider applicationCertificateProvider;
     private final MavenRepository mavenRepository;
     private final MeteringClient meteringClient;
+    private final GlobalRoutingService globalRoutingService;
 
     /**
      * Creates a controller 
@@ -98,12 +100,13 @@ public class Controller extends AbstractComponent {
                       AccessControl accessControl,
                       ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud,
                       BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource,
-                      MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider, MeteringClient meteringClient) {
+                      MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider,
+                      MeteringClient meteringClient, GlobalRoutingService globalRoutingService) {
         this(curator, rotationsConfig, zoneRegistry,
              configServer, metricsService, routingGenerator,
              Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud,
              buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource,
-             mavenRepository, applicationCertificateProvider, meteringClient);
+             mavenRepository, applicationCertificateProvider, meteringClient, globalRoutingService);
     }
 
     public Controller(CuratorDb curator, RotationsConfig rotationsConfig,
@@ -113,7 +116,9 @@ public class Controller extends AbstractComponent {
                       AccessControl accessControl,
                       ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud,
                       BuildService buildService, RunDataStore runDataStore, Supplier<String> hostnameSupplier,
-                      Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider, MeteringClient meteringClient) {
+                      Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository,
+                      ApplicationCertificateProvider applicationCertificateProvider, MeteringClient meteringClient,
+                      GlobalRoutingService globalRoutingService) {
 
         this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null");
         this.curator = Objects.requireNonNull(curator, "Curator cannot be null");
@@ -123,11 +128,12 @@ public class Controller extends AbstractComponent {
         this.clock = Objects.requireNonNull(clock, "Clock cannot be null");
         this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null");
         this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null");
-        this.nameServiceForwarder = new NameServiceForwarder(curator);
         this.applicationCertificateProvider = Objects.requireNonNull(applicationCertificateProvider);
         this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null");
-        this.meteringClient = meteringClient;
+        this.meteringClient = Objects.requireNonNull(meteringClient, "MeteringClient cannot be null");
+        this.globalRoutingService = Objects.requireNonNull(globalRoutingService, "GlobalRoutingSerivce cannot be null");
 
+        nameServiceForwarder = new NameServiceForwarder(curator);
         jobController = new JobController(this, runDataStore, Objects.requireNonNull(testerCloud));
         applicationController = new ApplicationController(this, curator, accessControl,
                                                           Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"),
@@ -165,6 +171,10 @@ public class Controller extends AbstractComponent {
         return flagSource;
     }
 
+    public GlobalRoutingService globalRoutingService() {
+        return globalRoutingService;
+    }
+
     public Clock clock() { return clock; }
 
     public ZoneRegistry zoneRegistry() { return zoneRegistry; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 2afbeb7b873..d76f120dddd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -6,16 +6,15 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
-import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
+import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService.ApplicationMetrics;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
@@ -24,7 +23,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Instant;
 import java.util.LinkedHashMap;
@@ -58,7 +57,7 @@ public class LockedApplication {
     private final ApplicationMetrics metrics;
     private final Optional<String> pemDeployKey;
     private final List<AssignedRotation> rotations;
-    private final Map<HostName, RotationState> rotationStatus;
+    private final RotationStatus rotationStatus;
     private final Optional<ApplicationCertificate> applicationCertificate;
 
     /**
@@ -81,7 +80,7 @@ public class LockedApplication {
                               Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                               Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                               OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
-                              List<AssignedRotation> rotations, Map<HostName, RotationState> rotationStatus, Optional<ApplicationCertificate> applicationCertificate) {
+                              List<AssignedRotation> rotations, RotationStatus rotationStatus, Optional<ApplicationCertificate> applicationCertificate) {
         this.lock = lock;
         this.id = id;
         this.createdAt = createdAt;
@@ -266,7 +265,7 @@ public class LockedApplication {
                                      metrics, pemDeployKey, assignedRotations, rotationStatus, applicationCertificate);
     }
 
-    public LockedApplication withRotationStatus(Map<HostName, RotationState> rotationStatus) {
+    public LockedApplication withRotationStatus(RotationStatus rotationStatus) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
                                      metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index b3f6a9546ca..debf07ea127 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -1,8 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -10,13 +10,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -102,15 +102,16 @@ public class DeploymentMetricsMaintainer extends Maintainer {
     }
 
     /** Get global rotation status for application */
-    private Map<HostName, RotationState> rotationStatus(Application application) {
+    // TODO(mpolden): Stop fetching rotation status from MetricsService and remove this
+    private RotationStatus rotationStatus(Application application) {
         return applications.rotationRepository().getRotation(application)
-                           .map(rotation -> controller().metricsService().getRotationStatus(rotation.name()))
-                           .map(rotationStatus -> {
-                               Map<HostName, RotationState> result = new TreeMap<>();
-                               rotationStatus.forEach((hostname, status) -> result.put(hostname, from(status)));
-                               return result;
+                           .map(rotation -> {
+                               var rotationStatus = controller().metricsService().getRotationStatus(rotation.name());
+                               var statusMap = new LinkedHashMap<ZoneId, RotationState>();
+                               rotationStatus.forEach((hostname, zoneStatus) -> statusMap.put(ZoneId.from("prod", hostname.value()), from(zoneStatus)));
+                               return new RotationStatus(Map.of(rotation.id(), statusMap));
                            })
-                           .orElseGet(Collections::emptyMap);
+                           .orElse(RotationStatus.EMPTY);
     }
 
     private static RotationState from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) {
@@ -121,4 +122,5 @@ public class DeploymentMetricsMaintainer extends Maintainer {
             default: throw new IllegalArgumentException("Unknown API value for rotation status: " + status);
         }
     }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index c39232cefca..ad7ef2148cd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -32,8 +32,9 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Instant;
 import java.util.ArrayList;
@@ -84,7 +85,7 @@ public class ApplicationSerializer {
     private static final String assignedRotationEndpointField = "endpointId";
     private static final String assignedRotationClusterField = "clusterId";
     private static final String assignedRotationRotationField = "rotationId";
-    private static final String rotationStatusField = "rotationStatus";
+    private static final String legacyRotationStatusField = "rotationStatus";
     private static final String applicationCertificateField = "applicationCertificate";
 
     // Deployment fields
@@ -157,6 +158,13 @@ public class ApplicationSerializer {
     private static final String deploymentMetricsUpdateTime = "lastUpdated";
     private static final String deploymentMetricsWarningsField = "warnings";
 
+    // RotationStatus fields
+    private static final String rotationStatusField = "rotationStatus2";
+    private static final String rotationIdField = "rotationId";
+    private static final String rotationStateField = "state";
+    private static final String statusField = "status";
+    private static final String hostnameField = "hostname";
+
     // ------------------ Serialization
 
     public Slime toSlime(Application application) {
@@ -178,6 +186,15 @@ public class ApplicationSerializer {
         application.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey));
         assignedRotationsToSlime(application.rotations(), root, assignedRotationsField);
         toSlime(application.rotationStatus(), root.setArray(rotationStatusField));
+        { // TODO(mpolden): Remove this block after September 2019
+            var firstRotationStatus = application.rotations().stream().findFirst()
+                                                 .map(AssignedRotation::rotationId)
+                                                 .flatMap(rotation -> application.rotationStatus().asMap().entrySet().stream()
+                                                                                 .filter(kv -> kv.getKey().equals(rotation))
+                                                                                 .map(Map.Entry::getValue).findFirst())
+                                                 .orElse(Map.of());
+            legacyToSlime(firstRotationStatus, root.setArray(legacyRotationStatusField));
+        }
         application.applicationCertificate().ifPresent(cert -> root.setString(applicationCertificateField, cert.secretsKeyNamePrefix()));
         return slime;
     }
@@ -317,11 +334,24 @@ public class ApplicationSerializer {
             object.setBool(pinnedField, true);
     }
 
-    private void toSlime(Map<HostName, RotationState> rotationStatus, Cursor array) {
-        rotationStatus.forEach((hostname, status) -> {
+    private void toSlime(RotationStatus status, Cursor array) {
+        status.asMap().forEach((rotationId, zoneStatus) -> {
+            Cursor rotationObject = array.addObject();
+            rotationObject.setString(rotationIdField, rotationId.asString());
+            Cursor statusArray = rotationObject.setArray(statusField);
+            zoneStatus.forEach((zone, state) -> {
+                Cursor statusObject = statusArray.addObject();
+                zoneIdToSlime(zone, statusObject);
+                statusObject.setString(rotationStateField, state.name());
+            });
+        });
+    }
+
+    private void legacyToSlime(Map<ZoneId, RotationState> state, Cursor array) {
+        state.forEach((zone, status) -> {
             Cursor object = array.addObject();
-            object.setString("hostname", hostname.value());
-            object.setString("status", status.name());
+            object.setString(hostnameField, zone.value());
+            object.setString(statusField, status.name());
         });
     }
 
@@ -355,7 +385,7 @@ public class ApplicationSerializer {
                                                             root.field(writeQualityField).asDouble());
         Optional<String> pemDeployKey = Serializers.optionalString(root.field(pemDeployKeyField));
         List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(deploymentSpec, root);
-        Map<HostName, RotationState> rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField));
+        RotationStatus rotationStatus = rotationStatusFromSlime(root, assignedRotations.stream().findFirst());
         Optional<ApplicationCertificate> applicationCertificate = Serializers.optionalString(root.field(applicationCertificateField)).map(ApplicationCertificate::new);
 
         return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs,
@@ -403,19 +433,54 @@ public class ApplicationSerializer {
         return Collections.unmodifiableMap(warnings);
     }
 
-    private Map<HostName, RotationState> rotationStatusFromSlime(Inspector object) {
+    private RotationStatus rotationStatusFromSlime(Inspector parentObject, Optional<AssignedRotation> firstRotation) {
+        var object = parentObject.field(rotationStatusField);
+        if (firstRotation.isEmpty()) {
+            return RotationStatus.EMPTY;
+        }
+        if (!object.valid()) {
+            // TODO(mpolden): Remove compatibility after September 2019
+            var legacyRotationStatus = legacyRotationStatusFromSlime(parentObject.field(legacyRotationStatusField));
+            var status = new LinkedHashMap<ZoneId, RotationState>();
+            for (var kv : legacyRotationStatus.entrySet()) {
+                // Old format stores hostname instead of zone, but this is only used in substring matching so we can get
+                // away with this hack
+                status.put(ZoneId.from("prod", kv.getKey().value()), kv.getValue());
+            }
+            return new RotationStatus(Map.of(firstRotation.get().rotationId(), status));
+        }
+        var statusMap = new LinkedHashMap<RotationId, Map<ZoneId, RotationState>>();
+        object.traverse((ArrayTraverser) (idx, statusObject) -> statusMap.put(new RotationId(statusObject.field(rotationIdField).asString()),
+                                                                              singleRotationStatusFromSlime(statusObject.field(statusField))));
+        return new RotationStatus(statusMap);
+    }
+
+    private Map<HostName, RotationState> legacyRotationStatusFromSlime(Inspector object) {
         if (!object.valid()) {
             return Collections.emptyMap();
         }
         Map<HostName, RotationState> rotationStatus = new TreeMap<>();
         object.traverse((ArrayTraverser) (idx, inspect) -> {
-            HostName hostname = HostName.from(inspect.field("hostname").asString());
-            RotationState status = RotationState.valueOf(inspect.field("status").asString());
+            HostName hostname = HostName.from(inspect.field(hostnameField).asString());
+            RotationState status = RotationState.valueOf(inspect.field(statusField).asString());
             rotationStatus.put(hostname, status);
         });
         return Collections.unmodifiableMap(rotationStatus);
     }
 
+    private Map<ZoneId, RotationState> singleRotationStatusFromSlime(Inspector object) {
+        if (!object.valid()) {
+            return Collections.emptyMap();
+        }
+        Map<ZoneId, RotationState> rotationStatus = new LinkedHashMap<>();
+        object.traverse((ArrayTraverser) (idx, statusObject) -> {
+            var zone = zoneIdFromSlime(statusObject);
+            var status = RotationState.valueOf(statusObject.field(rotationStateField).asString());
+            rotationStatus.put(zone, status);
+        });
+        return Collections.unmodifiableMap(rotationStatus);
+    }
+
     private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime   (Inspector object) {
         Map<ClusterSpec.Id, ClusterInfo> map = new HashMap<>();
         object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterInfoFromSlime(value)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index a396088a9c4..40fdd193be9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -551,7 +551,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             Cursor deploymentObject = instancesArray.addObject();
 
             if (!application.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) {
-                toSlime(application.rotationStatus(deployment), deploymentObject);
+                toSlime(application.rotationStatus().of(deployment), deploymentObject);
             }
 
             if (recurseOverDeployments(request)) // List full deployment information when recursive.
@@ -776,7 +776,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         Slime slime = new Slime();
         Cursor response = slime.setObject();
-        toSlime(application.rotationStatus(deployment), response);
+        toSlime(application.rotationStatus().of(deployment), response);
         return new SlimeJsonResponse(slime);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
new file mode 100644
index 00000000000..101c224b172
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
@@ -0,0 +1,66 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.rotation;
+
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * The status of all rotations assigned to an application.
+ *
+ * @author mpolden
+ */
+public class RotationStatus {
+
+    public static final RotationStatus EMPTY = new RotationStatus(Map.of());
+
+    private final Map<RotationId, Map<ZoneId, RotationState>> status;
+
+    /** DO NOT USE. Public for serialization purposes */
+    public RotationStatus(Map<RotationId, Map<ZoneId, RotationState>> status) {
+        this.status = Map.copyOf(Objects.requireNonNull(status));
+    }
+
+    public Map<RotationId, Map<ZoneId, RotationState>> asMap() {
+        return status;
+    }
+
+    /** Get status of given rotation, if any */
+    public Map<ZoneId, RotationState> of(RotationId rotation) {
+        return status.getOrDefault(rotation, Map.of());
+    }
+
+    /** Get status of given deployment, if any */
+    public RotationState of(Deployment deployment) {
+        return status.values().stream()
+                     .map(Map::entrySet)
+                     .flatMap(Collection::stream)
+                     // TODO(mpolden): Change to exact comparison after September 2019
+                     .filter(kv -> kv.getKey().value().contains(deployment.zone().value()))
+                     .map(Map.Entry::getValue)
+                     .findFirst()
+                     .orElse(RotationState.unknown);
+    }
+
+    @Override
+    public String toString() {
+        return "rotation status " + status;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RotationStatus that = (RotationStatus) o;
+        return status.equals(that.status);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(status);
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 7c5442edbde..b79c0dc4ac2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
@@ -349,7 +350,8 @@ public final class ControllerTester {
                                                new InMemoryFlagSource(),
                                                new MockMavenRepository(),
                                                new ApplicationCertificateMock(),
-                                               new MockMeteringClient());
+                                               new MockMeteringClient(),
+                                               new MemoryGlobalRoutingService());
         // Calculate initial versions
         controller.updateVersionStatus(VersionStatus.compute(controller));
         return controller;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index d75bb283fe6..ccafab622d0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -115,21 +115,21 @@ public class DeploymentMetricsMaintainerTest {
         tester.controllerTester().metricsService().addRotation(assignedRotation);
 
         // No status gathered yet
-        assertEquals(RotationState.unknown, app.get().rotationStatus(deployment1.get()));
-        assertEquals(RotationState.unknown, app.get().rotationStatus(deployment2.get()));
+        assertEquals(RotationState.unknown, app.get().rotationStatus().of(deployment1.get()));
+        assertEquals(RotationState.unknown, app.get().rotationStatus().of(deployment2.get()));
 
         // One rotation out, one in
         metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test");
         metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test");
         maintainer.maintain();
-        assertEquals(RotationState.in, app.get().rotationStatus(deployment1.get()));
-        assertEquals(RotationState.out, app.get().rotationStatus(deployment2.get()));
+        assertEquals(RotationState.in, app.get().rotationStatus().of(deployment1.get()));
+        assertEquals(RotationState.out, app.get().rotationStatus().of(deployment2.get()));
 
         // All rotations in
         metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test");
         maintainer.maintain();
-        assertEquals(RotationState.in, app.get().rotationStatus(deployment1.get()));
-        assertEquals(RotationState.in, app.get().rotationStatus(deployment2.get()));
+        assertEquals(RotationState.in, app.get().rotationStatus().of(deployment1.get()));
+        assertEquals(RotationState.in, app.get().rotationStatus().of(deployment2.get()));
     }
 
     private static DeploymentMetricsMaintainer maintainer(Controller controller) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 36b13bbcf42..1f39840dc3a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -6,7 +6,6 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
@@ -27,9 +26,12 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
+import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 import org.junit.Test;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -37,6 +39,7 @@ import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -44,12 +47,12 @@ import java.util.OptionalDouble;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
-import java.util.TreeMap;
 
 import static com.yahoo.config.provision.SystemName.main;
 import static com.yahoo.vespa.hosted.controller.ControllerTester.writable;
 import static java.util.Optional.empty;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author bratseth
@@ -101,9 +104,10 @@ public class ApplicationSerializerTest {
 
         DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, empty(), true);
 
-        Map<HostName, RotationState> rotationStatus = new TreeMap<>();
-        rotationStatus.put(HostName.from("rot1.fqdn"), RotationState.in);
-        rotationStatus.put(HostName.from("rot2.fqdn"), RotationState.out);
+        var rotationStatusMap = new LinkedHashMap<ZoneId, RotationState>();
+        rotationStatusMap.put(ZoneId.from("prod", "us-west-1"), RotationState.in);
+        rotationStatusMap.put(ZoneId.from("prod", "us-east-3"), RotationState.out);
+        var rotationStatus = new RotationStatus(Map.of(new RotationId("my-rotation"), rotationStatusMap));
 
         Application original = new Application(ApplicationId.from("t1", "a1", "i1"),
                                                Instant.now().truncatedTo(ChronoUnit.MILLIS),
@@ -249,4 +253,20 @@ public class ApplicationSerializerTest {
         // ok if no error
     }
 
+    @Test // TODO(mpolden): Remove after september 2019
+    public void testLegacyRotationStatus() throws Exception {
+        var json = Files.readAllBytes(testData.resolve("complete-application.json"));
+        var application = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(json));
+        var expected = new RotationStatus(Map.of(new RotationId("rotation-foo"),
+                                                 Map.of(ZoneId.from("prod", "host1.fqdn"), RotationState.out,
+                                                        ZoneId.from("prod", "host2.fqdn"), RotationState.in)));
+        assertEquals(expected, application.rotationStatus());
+
+        // Writes both new and old format
+        var serializedJson = new String(SlimeUtils.toJsonBytes(applicationSerializer.toSlime(application)), StandardCharsets.UTF_8);
+        var jsonFragment = "\"rotationStatus2\":[{\"rotationId\":\"rotation-foo\",\"status\":[{\"environment\":\"prod\",\"region\":\"host1.fqdn\",\"state\":\"out\"},{\"environment\":\"prod\",\"region\":\"host2.fqdn\",\"state\":\"in\"}]}]," +
+                           "\"rotationStatus\":[{\"hostname\":\"prod.host1.fqdn\",\"status\":\"out\"},{\"hostname\":\"prod.host2.fqdn\",\"status\":\"in\"}]}";
+        assertTrue("Writes both new and old format", serializedJson.contains(jsonFragment));
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
index cb5e34b8dae..c43b666d636 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json
@@ -511,5 +511,22 @@
   "outstandingChangeField": false,
   "queryQuality": 100,
   "writeQuality": 99.99894341115082,
-  "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"
+  "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----",
+  "assignedRotations": [
+    {
+      "rotationId": "rotation-foo",
+      "clusterId": "qrs",
+      "endpointId": "default"
+    }
+  ],
+  "rotationStatus": [
+    {
+      "status": "out",
+      "hostname": "host1.fqdn"
+    },
+    {
+      "status": "in",
+      "hostname": "host2.fqdn"
+    }
+  ]
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index fe31200dc93..f2fbf36bec2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -66,6 +66,7 @@ public class ControllerContainerTest {
                "  <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.restapi.cost.NoopCostReportConsumer'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues'/>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 6ce7d7202e8..9bc5c879961 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -49,7 +49,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction;
 import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities;
@@ -63,6 +62,8 @@ import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
@@ -79,11 +80,11 @@ import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.TreeMap;
 import java.util.function.Supplier;
 
 import static com.yahoo.application.container.handler.Request.Method.DELETE;
@@ -1677,17 +1678,18 @@ public class ApplicationApiTest extends ControllerContainerTest {
         applicationList.forEach(application -> {
                 applicationController.lockIfPresent(application.id(), locked ->
                         applicationController.store(locked.withRotationStatus(rotationStatus(application))));
-        });}
+        });
+    }
 
-    private Map<HostName, RotationState> rotationStatus(Application application) {
+    private RotationStatus rotationStatus(Application application) {
         return controllerTester.controller().applications().rotationRepository().getRotation(application)
-                .map(rotation -> controllerTester.controller().metricsService().getRotationStatus(rotation.name()))
-                .map(rotationStatus -> {
-                    Map<HostName, RotationState> result = new TreeMap<>();
-                    rotationStatus.forEach((hostname, status) -> result.put(hostname, RotationState.in));
-                    return result;
+                .map(rotation -> {
+                    var rotationStatus = controllerTester.controller().metricsService().getRotationStatus(rotation.name());
+                    var statusMap = new LinkedHashMap<ZoneId, RotationState>();
+                    rotationStatus.forEach((hostname, status) -> statusMap.put(ZoneId.from("prod", hostname.value()), RotationState.in));
+                    return new RotationStatus(Map.of(rotation.id(), statusMap));
                 })
-                .orElseGet(Collections::emptyMap);
+                .orElse(RotationStatus.EMPTY);
     }
 
     private void updateContactInformation() {