Support AssignedRotations that track cluster, rotation, and endpoint name.

No functionality changes in this commit, only how we persist and talk about
rotations and endpoints in the applicaiton.  This is preparation of actually
updating the support for multiple endpoints in the controller.

6 files changed, 208 insertions, 46 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index b4e3b8b1a9a..0a85e68f4b6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -5,9 +5,11 @@ import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
+import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RotationName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics;
@@ -15,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.ApplicationActivity;
+import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
@@ -27,14 +30,17 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import java.time.Instant;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.OptionalLong;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * An instance of an application.
@@ -58,8 +64,7 @@ public class Application {
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
     private final Optional<String> pemDeployKey;
-    private final Optional<RotationId> legacyRotation;
-    private final List<RotationId> rotations;
+    private final List<AssignedRotation> rotations;
     private final Map<HostName, RotationStatus> rotationStatus;
 
     /** Creates an empty application */
@@ -68,7 +73,7 @@ public class Application {
              new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false),
              Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(),
              new ApplicationMetrics(0, 0),
-             Optional.empty(), Optional.empty(), Collections.emptyList(), Collections.emptyMap());
+             Optional.empty(), Collections.emptyList(), Collections.emptyMap());
     }
 
     /** Used from persistence layer: Do not use */
@@ -76,18 +81,18 @@ public class Application {
                        List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                        Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                        OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
-                       Optional<RotationId> legacyRotation, List<RotationId> rotations, Map<HostName, RotationStatus> rotationStatus) {
+                       List<AssignedRotation> rotations, Map<HostName, RotationStatus> rotationStatus) {
         this(id, createdAt, deploymentSpec, validationOverrides,
              deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())),
              deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-             metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+             metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     Application(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides,
                 Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                 Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                 OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
-                Optional<RotationId> legacyRotation, List<RotationId> rotations, Map<HostName, RotationStatus> rotationStatus) {
+                List<AssignedRotation> rotations, Map<HostName, RotationStatus> rotationStatus) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null");
         this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null");
@@ -101,7 +106,6 @@ public class Application {
         this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null");
         this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null");
         this.pemDeployKey = pemDeployKey;
-        this.legacyRotation = Objects.requireNonNull(legacyRotation, "legacyRotation cannot be null");
         this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null"));
         this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null"));
     }
@@ -200,11 +204,20 @@ public class Application {
 
     /** Returns the global rotation id of this, if present */
     public Optional<RotationId> legacyRotation() {
-        return legacyRotation;
+        return rotations.stream()
+                .map(AssignedRotation::rotationId)
+                .findFirst();
     }
 
     /** Returns all rotations for this application */
     public List<RotationId> rotations() {
+        return rotations.stream()
+                .map(AssignedRotation::rotationId)
+                .collect(Collectors.toList());
+    }
+
+    /** Returns all assigned rotations for this application */
+    public List<AssignedRotation> assignedRotations() {
         return rotations;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 782a76b684e..8345aa91685 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -43,9 +43,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.application.JobList;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
@@ -439,7 +441,7 @@ public class ApplicationController {
         if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) {
             try (RotationLock rotationLock = rotationRepository.lock()) {
                 Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock);
-                application = application.with(rotation.id());
+                application = application.with(List.of(new AssignedRotation(new ClusterSpec.Id(application.get().deploymentSpec().globalServiceId().get()), EndpointId.default_(), rotation.id())));
                 store(application); // store assigned rotation even if deployment fails
 
                 boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 5f958b74c39..42162784957 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
@@ -56,8 +57,7 @@ public class LockedApplication {
     private final OptionalInt majorVersion;
     private final ApplicationMetrics metrics;
     private final Optional<String> pemDeployKey;
-    private final Optional<RotationId> legacyRotation;
-    private final List<RotationId> rotations;
+    private final List<AssignedRotation> rotations;
     private final Map<HostName, RotationStatus> rotationStatus;
 
     /**
@@ -72,7 +72,7 @@ public class LockedApplication {
              application.deployments(),
              application.deploymentJobs(), application.change(), application.outstandingChange(),
              application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(),
-             application.pemDeployKey(), application.legacyRotation(), application.rotations(), application.rotationStatus());
+             application.pemDeployKey(), application.assignedRotations(), application.rotationStatus());
     }
 
     private LockedApplication(Lock lock, ApplicationId id, Instant createdAt,
@@ -80,7 +80,7 @@ public class LockedApplication {
                               Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change,
                               Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner,
                               OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey,
-                              Optional<RotationId> legacyRotation, List<RotationId> rotations, Map<HostName, RotationStatus> rotationStatus) {
+                              List<AssignedRotation> rotations, Map<HostName, RotationStatus> rotationStatus) {
         this.lock = lock;
         this.id = id;
         this.createdAt = createdAt;
@@ -95,7 +95,6 @@ public class LockedApplication {
         this.majorVersion = majorVersion;
         this.metrics = metrics;
         this.pemDeployKey = pemDeployKey;
-        this.legacyRotation = legacyRotation;
         this.rotations = rotations;
         this.rotationStatus = rotationStatus;
     }
@@ -104,35 +103,35 @@ public class LockedApplication {
     public Application get() {
         return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change,
                                outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                               legacyRotation, rotations, rotationStatus);
+                               rotations, rotationStatus);
     }
 
     public LockedApplication withBuiltInternally(boolean builtInternally) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication withProjectId(OptionalLong projectId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withProjectId(projectId), change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication withDeploymentIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.with(issueId), change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion,
@@ -140,14 +139,14 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withCompletion(projectId, jobType, completion, jobError),
                                      change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics,
-                                     pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.withTriggering(jobType, job), change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version,
@@ -198,45 +197,45 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs.without(jobType), change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication with(DeploymentSpec deploymentSpec) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange,
                                      ownershipIssueId, owner, majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     public LockedApplication with(ValidationOverrides validationOverrides) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication withChange(Change change) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication withOutstandingChange(Change outstandingChange) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication withOwnershipIssueId(IssueId issueId) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, Optional.ofNullable(issueId), owner,
-                                     majorVersion, metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     majorVersion, metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication withOwner(User owner) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId,
                                      Optional.ofNullable(owner), majorVersion, metrics, pemDeployKey,
-                                     legacyRotation, rotations, rotationStatus);
+                                     rotations, rotationStatus);
     }
 
     /** Set a major version for this, or set to null to remove any major version override */
@@ -244,31 +243,31 @@ public class LockedApplication {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner,
                                      majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion),
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication with(MetricsService.ApplicationMetrics metrics) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     public LockedApplication withPemDeployKey(String pemDeployKey) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, Optional.ofNullable(pemDeployKey), legacyRotation, rotations, rotationStatus);
+                                     metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus);
     }
 
-    public LockedApplication with(RotationId rotation) {
+    public LockedApplication with(List<AssignedRotation> assignedRotations) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
-                                     deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, Optional.of(rotation), List.of(rotation), rotationStatus);
+                deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
+                metrics, pemDeployKey, assignedRotations, rotationStatus);
     }
 
     public LockedApplication withRotationStatus(Map<HostName, RotationStatus> rotationStatus) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     /** Don't expose non-leaf sub-objects. */
@@ -281,7 +280,7 @@ public class LockedApplication {
     private LockedApplication with(Map<ZoneId, Deployment> deployments) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
-                                     metrics, pemDeployKey, legacyRotation, rotations, rotationStatus);
+                                     metrics, pemDeployKey, rotations, rotationStatus);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java
new file mode 100644
index 00000000000..e1ed278a79e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java
@@ -0,0 +1,61 @@
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.vespa.hosted.controller.rotation.RotationId;
+
+import java.util.Objects;
+
+/**
+ * Contains the tuple of [clusterId, endpointId, rotationId], to keep track
+ * of which services have assigned which rotations under which name.
+ *
+ * @author ogronnesby
+ */
+public class AssignedRotation {
+    private final ClusterSpec.Id clusterId;
+    private final EndpointId endpointId;
+    private final RotationId rotationId;
+
+    public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId) {
+        this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId");
+        this.endpointId = Objects.requireNonNull(endpointId);
+        this.rotationId = Objects.requireNonNull(rotationId);
+    }
+
+    public ClusterSpec.Id clusterId() { return clusterId; }
+    public EndpointId endpointId() { return endpointId; }
+    public RotationId rotationId() { return rotationId; }
+
+    @Override
+    public String toString() {
+        return "AssignedRotation{" +
+                "clusterId=" + clusterId +
+                ", endpointId='" + endpointId + '\'' +
+                ", rotationId=" + rotationId +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AssignedRotation that = (AssignedRotation) o;
+        return clusterId.equals(that.clusterId) &&
+                endpointId.equals(that.endpointId) &&
+                rotationId.equals(that.rotationId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(clusterId, endpointId, rotationId);
+    }
+
+    private static <T> T requireNonEmpty(T object, String value, String field) {
+        Objects.requireNonNull(object);
+        Objects.requireNonNull(value);
+        if (value.isEmpty()) {
+            throw new IllegalArgumentException("Field '" + field + "' was empty");
+        }
+        return object;
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java
new file mode 100644
index 00000000000..f186125d09a
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java
@@ -0,0 +1,47 @@
+package com.yahoo.vespa.hosted.controller.application;
+
+import java.util.Objects;
+
+public class EndpointId {
+    private static final EndpointId DEFAULT = new EndpointId("default");
+
+    private final String id;
+
+    public EndpointId(String id) {
+        this.id = requireNotEmpty(id);
+    }
+
+    public String id() { return id; }
+
+    @Override
+    public String toString() {
+        return "EndpointId{" +
+                "id='" + id + '\'' +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EndpointId that = (EndpointId) o;
+        return Objects.equals(id, that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+    private static String requireNotEmpty(String input) {
+        Objects.requireNonNull(input);
+        if (input.isEmpty()) {
+            throw new IllegalArgumentException("The value EndpointId was empty");
+        }
+        return input;
+    }
+
+    public static EndpointId default_() { return DEFAULT; }
+
+    public static EndpointId of(String id) { return new EndpointId(id); }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 1f20bdf5533..949b7d000d9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.document.datatypes.Array;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
@@ -21,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
@@ -29,6 +31,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
@@ -38,6 +41,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -77,6 +81,7 @@ public class ApplicationSerializer {
     private final String writeQualityField = "writeQuality";
     private final String queryQualityField = "queryQuality";
     private final String pemDeployKeyField = "pemDeployKey";
+    private final String assignedRotationsField = "assignedRotations";
     private final String rotationsField = "endpoints";
     private final String deprecatedRotationField = "rotation";
     private final String rotationStatusField = "rotationStatus";
@@ -171,8 +176,8 @@ public class ApplicationSerializer {
         root.setDouble(writeQualityField, application.metrics().writeServiceQuality());
         application.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey));
         application.legacyRotation().ifPresent(rotation -> root.setString(deprecatedRotationField, rotation.asString()));
-        Cursor rotations = root.setArray(rotationsField);
-        application.rotations().forEach(rotation -> rotations.addString(rotation.asString()));
+        rotationsToSlime(application.assignedRotations(), root, rotationsField);
+        assignedRotationsToSlime(application.assignedRotations(), root, assignedRotationsField);
         toSlime(application.rotationStatus(), root.setArray(rotationStatusField));
         return slime;
     }
@@ -320,6 +325,21 @@ public class ApplicationSerializer {
         });
     }
 
+    private void rotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) {
+        final var rotationsArray = parent.setArray(fieldName);
+        rotations.forEach(rot -> rotationsArray.addString(rot.rotationId().asString()));
+    }
+
+    private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) {
+        final var rotationsArray = parent.setArray(fieldName);
+        for (var rotation : rotations) {
+            final var object = rotationsArray.addObject();
+            object.setString("endpoint", rotation.endpointId().id());
+            object.setString("rotation", rotation.rotationId().asString());
+            object.setString("container", rotation.clusterId().value());
+        }
+    }
+
     // ------------------ Deserialization
 
     public Application fromSlime(Slime slime) {
@@ -339,13 +359,12 @@ public class ApplicationSerializer {
         ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
                                                             root.field(writeQualityField).asDouble());
         Optional<String> pemDeployKey = optionalString(root.field(pemDeployKeyField));
-        Optional<RotationId> legacyRotation = optionalString(root.field(deprecatedRotationField)).map(RotationId::new);
-        List<RotationId> rotations = rotationsFromSlime(root);
+        List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(deploymentSpec, root);
         Map<HostName, RotationStatus> rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField));
 
         return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs,
                                deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics,
-                               pemDeployKey, legacyRotation, rotations, rotationStatus);
+                               pemDeployKey, assignedRotations, rotationStatus);
     }
 
     private List<Deployment> deploymentsFromSlime(Inspector array) {
@@ -525,15 +544,36 @@ public class ApplicationSerializer {
                                                 Instant.ofEpochMilli(object.field(atField).asLong())));
     }
 
-    private List<RotationId> rotationsFromSlime(Inspector root) {
-        final var rotations = rotationListFromSlime(root.field(rotationsField));
+    private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) {
+        final var assignedRotations = new LinkedHashSet<AssignedRotation>();
+
+        // Add the legacy rotation field to the set - this needs to be first
+        // TODO: Remove when we retire the rotations field
         final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField));
+        if (legacyRotation.isPresent() && deploymentSpec.globalServiceId().isPresent()) {
+            final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get());
+            assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get()));
+        }
 
-        if (legacyRotation.isPresent() && ! rotations.contains(legacyRotation.get())) {
-            rotations.add(legacyRotation.get());
+        // Now add the same entries from "stupid" list of rotations
+        // TODO: Remove when we retire the rotations field
+        final var rotations = rotationListFromSlime(root.field(rotationsField));
+        for (var rotation : rotations) {
+            if (deploymentSpec.globalServiceId().isPresent()) {
+                final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get());
+                assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), rotation));
+            }
         }
 
-        return rotations;
+        // Last - add the actual entries we want.  Do _not_ remove this during clean-up
+        root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> {
+            final var clusterId = new ClusterSpec.Id(inspector.field("container").asString());
+            final var endpointId = EndpointId.of(inspector.field("endpoint").asString());
+            final var rotationId = new RotationId(inspector.field("rotation").asString());
+            assignedRotations.add(new AssignedRotation(clusterId, endpointId, rotationId));
+        });
+
+        return List.copyOf(assignedRotations);
     }
 
     private List<RotationId> rotationListFromSlime(Inspector field) {