Retrieve rotation status from GlobalRoutingService

12 files changed, 243 insertions, 100 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index d76f120dddd..8ea4ddeb631 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -265,7 +265,7 @@ public class LockedApplication {
                                      metrics, pemDeployKey, assignedRotations, rotationStatus, applicationCertificate);
     }
 
-    public LockedApplication withRotationStatus(RotationStatus rotationStatus) {
+    public LockedApplication with(RotationStatus rotationStatus) {
         return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments,
                                      deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion,
                                      metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index dec1847e751..56e77f3254b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -165,6 +165,11 @@ public class ApplicationList {
         return listOf(list.stream().filter(a -> a.deploymentSpec().canUpgradeAt(instant)));
     }
 
+    /** Returns the subset of applications that have at least one assigned rotation */
+    public ApplicationList hasRotation() {
+        return listOf(list.stream().filter(a -> !a.rotations().isEmpty()));
+    }
+
     /**
      * Returns the subset of applications that hasn't pinned to an an earlier major version than the given one.
      *
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 4cadefc35b5..ec73ea62465 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -54,6 +54,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final NameServiceDispatcher nameServiceDispatcher;
     private final BillingMaintainer billingMaintainer;
     private final AwsEventReporterMaintainer awsEventReporterMaintainer;
+    private final RotationStatusUpdater rotationStatusUpdater;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
     public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator,
@@ -90,6 +91,7 @@ public class ControllerMaintenance extends AbstractComponent {
         nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService);
         billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing);
         awsEventReporterMaintainer = new AwsEventReporterMaintainer(controller, Duration.ofDays(1), jobControl, issueHandler, awsEventFetcher);
+        rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval, jobControl);
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -120,6 +122,7 @@ public class ControllerMaintenance extends AbstractComponent {
         nameServiceDispatcher.deconstruct();
         billingMaintainer.deconstruct();
         awsEventReporterMaintainer.deconstruct();
+        rotationStatusUpdater.deconstruct();
     }
 
     /** Create one OS upgrader per cloud found in the zone registry of controller */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index debf07ea127..74bb11f575d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -2,21 +2,16 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.metrics.MetricsService;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Duration;
 import java.time.Instant;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -58,9 +53,6 @@ public class DeploymentMetricsMaintainer extends Maintainer {
                     applications.lockIfPresent(application.id(), locked ->
                             applications.store(locked.with(controller().metricsService().getApplicationMetrics(application.id()))));
 
-                    applications.lockIfPresent(application.id(), locked ->
-                            applications.store(locked.withRotationStatus(rotationStatus(application))));
-
                     for (Deployment deployment : application.deployments().values()) {
                         MetricsService.DeploymentMetrics collectedMetrics = controller().metricsService()
                                                                                         .getDeploymentMetrics(application.id(), deployment.zone());
@@ -101,26 +93,8 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         }
     }
 
-    /** Get global rotation status for application */
-    // TODO(mpolden): Stop fetching rotation status from MetricsService and remove this
-    private RotationStatus rotationStatus(Application application) {
-        return applications.rotationRepository().getRotation(application)
-                           .map(rotation -> {
-                               var rotationStatus = controller().metricsService().getRotationStatus(rotation.name());
-                               var statusMap = new LinkedHashMap<ZoneId, RotationState>();
-                               rotationStatus.forEach((hostname, zoneStatus) -> statusMap.put(ZoneId.from("prod", hostname.value()), from(zoneStatus)));
-                               return new RotationStatus(Map.of(rotation.id(), statusMap));
-                           })
-                           .orElse(RotationStatus.EMPTY);
-    }
-
-    private static RotationState from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) {
-        switch (status) {
-            case IN: return RotationState.in;
-            case OUT: return RotationState.out;
-            case UNKNOWN: return RotationState.unknown;
-            default: throw new IllegalArgumentException("Unknown API value for rotation status: " + status);
-        }
+    private static DeploymentMetricsMaintainer maintainer(Controller controller) {
+        return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
new file mode 100644
index 00000000000..0bb98d7837c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
@@ -0,0 +1,97 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+import com.yahoo.vespa.hosted.controller.rotation.RotationId;
+import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
+
+import java.time.Duration;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+/**
+ * Periodically updates the status of assigned global rotations.
+ *
+ * @author mpolden
+ */
+public class RotationStatusUpdater extends Maintainer {
+
+    private static final int applicationsToUpdateInParallel = 10;
+
+    private final GlobalRoutingService service;
+    private final ApplicationController applications;
+
+    public RotationStatusUpdater(Controller controller, Duration interval, JobControl jobControl) {
+        super(controller, interval, jobControl);
+        this.service = controller.globalRoutingService();
+        this.applications = controller.applications();
+    }
+
+    @Override
+    protected void maintain() {
+        var failures = new AtomicInteger(0);
+        var lastException = new AtomicReference<Exception>(null);
+        var applicationList = ApplicationList.from(applications.asList()).hasRotation();
+
+        // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used
+        var pool = new ForkJoinPool(applicationsToUpdateInParallel);
+
+        pool.submit(() -> {
+            applicationList.asList().parallelStream().forEach(application -> {
+                try {
+                    applications.lockIfPresent(application.id(), (app) -> applications.store(app.with(getStatus(app.get()))));
+                } catch (Exception e) {
+                    failures.incrementAndGet();
+                    lastException.set(e);
+                }
+            });
+        });
+        pool.shutdown();
+        try {
+            pool.awaitTermination(30, TimeUnit.SECONDS);
+            if (lastException.get() != null) {
+                log.log(LogLevel.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: ",
+                                                        failures.get(),
+                                                        applicationList.size(),
+                                                        maintenanceInterval()),
+                        lastException.get());
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private RotationStatus getStatus(Application application) {
+        var statusMap = new LinkedHashMap<RotationId, Map<ZoneId, RotationState>>();
+        for (var assignedRotation : application.rotations()) {
+            var rotation = applications.rotationRepository().getRotation(assignedRotation.rotationId());
+            if (rotation.isEmpty()) continue;
+            var rotationStatus = service.getHealthStatus(rotation.get().name()).entrySet().stream()
+                                        .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue())));
+            statusMap.put(assignedRotation.rotationId(), rotationStatus);
+        }
+        return new RotationStatus(statusMap);
+    }
+
+    private static RotationState from(com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus status) {
+        switch (status) {
+            case IN: return RotationState.in;
+            case OUT: return RotationState.out;
+            case UNKNOWN: return RotationState.unknown;
+            default: throw new IllegalArgumentException("Unknown API value for rotation status: " + status);
+        }
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 40fdd193be9..7db75d73ea4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -551,7 +551,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             Cursor deploymentObject = instancesArray.addObject();
 
             if (!application.rotations().isEmpty() && deployment.zone().environment() == Environment.prod) {
-                toSlime(application.rotationStatus().of(deployment), deploymentObject);
+                // TODO(mpolden): Support retrieving status for multiple rotations
+                toSlime(application.rotationStatus().of(application.rotations().get(0).rotationId(), deployment), deploymentObject);
             }
 
             if (recurseOverDeployments(request)) // List full deployment information when recursive.
@@ -776,7 +777,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         Slime slime = new Slime();
         Cursor response = slime.setObject();
-        toSlime(application.rotationStatus().of(deployment), response);
+        // TODO(mpolden): Support retrieving status for multiple rotations
+        toSlime(application.rotationStatus().of(application.rotations().get(0).rotationId(), deployment), response);
         return new SlimeJsonResponse(slime);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
index 101c224b172..62750a5b608 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.rotation;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 
-import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 
@@ -19,7 +18,6 @@ public class RotationStatus {
 
     private final Map<RotationId, Map<ZoneId, RotationState>> status;
 
-    /** DO NOT USE. Public for serialization purposes */
     public RotationStatus(Map<RotationId, Map<ZoneId, RotationState>> status) {
         this.status = Map.copyOf(Objects.requireNonNull(status));
     }
@@ -33,16 +31,14 @@ public class RotationStatus {
         return status.getOrDefault(rotation, Map.of());
     }
 
-    /** Get status of given deployment, if any */
-    public RotationState of(Deployment deployment) {
-        return status.values().stream()
-                     .map(Map::entrySet)
-                     .flatMap(Collection::stream)
-                     // TODO(mpolden): Change to exact comparison after September 2019
-                     .filter(kv -> kv.getKey().value().contains(deployment.zone().value()))
-                     .map(Map.Entry::getValue)
-                     .findFirst()
-                     .orElse(RotationState.unknown);
+    /** Get status of deployment in given rotation, if any */
+    public RotationState of(RotationId rotation, Deployment deployment) {
+        return of(rotation).entrySet().stream()
+                           // TODO(mpolden): Change to exact comparison after September 2019
+                           .filter(kv -> kv.getKey().value().contains(deployment.zone().value()))
+                           .map(Map.Entry::getValue)
+                           .findFirst()
+                           .orElse(RotationState.unknown);
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index b79c0dc4ac2..3ce3ac5eecb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -90,6 +90,7 @@ public final class ControllerTester {
     private final MetricsServiceMock metricsService;
     private final RoutingGeneratorMock routingGenerator;
     private final MockContactRetriever contactRetriever;
+    private final MemoryGlobalRoutingService globalRoutingService;
 
     private Controller controller;
 
@@ -98,7 +99,7 @@ public final class ControllerTester {
         this(new AthenzDbMock(), clock, new ConfigServerMock(new ZoneRegistryMock()),
              new ZoneRegistryMock(), curatorDb, rotationsConfig,
              new MemoryNameService(), new ArtifactRepositoryMock(), new ApplicationStoreMock(), new MockBuildService(),
-             metricsService, new RoutingGeneratorMock(), new MockContactRetriever());
+             metricsService, new RoutingGeneratorMock(), new MockContactRetriever(), new MemoryGlobalRoutingService());
     }
 
     public ControllerTester(ManualClock clock) {
@@ -123,7 +124,7 @@ public final class ControllerTester {
                              MemoryNameService nameService, ArtifactRepositoryMock artifactRepository,
                              ApplicationStoreMock appStoreMock, MockBuildService buildService,
                              MetricsServiceMock metricsService, RoutingGeneratorMock routingGenerator,
-                             MockContactRetriever contactRetriever) {
+                             MockContactRetriever contactRetriever, MemoryGlobalRoutingService globalRoutingService) {
         this.athenzDb = athenzDb;
         this.clock = clock;
         this.configServer = configServer;
@@ -137,9 +138,10 @@ public final class ControllerTester {
         this.metricsService = metricsService;
         this.routingGenerator = routingGenerator;
         this.contactRetriever = contactRetriever;
+        this.globalRoutingService = globalRoutingService;
         this.controller = createController(curator, rotationsConfig, configServer, clock, zoneRegistry,
                                            athenzDb, artifactRepository, appStoreMock, buildService,
-                                           metricsService, routingGenerator);
+                                           metricsService, routingGenerator, globalRoutingService);
 
         // Make root logger use time from manual clock
         configureDefaultLogHandler(handler -> handler.setFilter(
@@ -189,6 +191,10 @@ public final class ControllerTester {
         return contactRetriever;
     }
 
+    public MemoryGlobalRoutingService globalRoutingService() {
+        return globalRoutingService;
+    }
+
     public Optional<Record> findCname(String name) {
         return nameService.findRecords(Record.Type.CNAME, RecordName.from(name)).stream().findFirst();
     }
@@ -197,7 +203,7 @@ public final class ControllerTester {
     public final void createNewController() {
         controller = createController(curator, rotationsConfig, configServer, clock, zoneRegistry, athenzDb,
                                       artifactRepository, applicationStore, buildService, metricsService,
-                                      routingGenerator);
+                                      routingGenerator, globalRoutingService);
     }
 
     /** Creates the given tenant and application and deploys it */
@@ -331,7 +337,8 @@ public final class ControllerTester {
                                                AthenzDbMock athensDb,
                                                ArtifactRepository artifactRepository, ApplicationStore applicationStore,
                                                BuildService buildService, MetricsServiceMock metricsService,
-                                               RoutingGenerator routingGenerator) {
+                                               RoutingGenerator routingGenerator,
+                                               MemoryGlobalRoutingService globalRoutingService) {
         Controller controller = new Controller(curator,
                                                rotationsConfig,
                                                zoneRegistryMock,
@@ -351,7 +358,7 @@ public final class ControllerTester {
                                                new MockMavenRepository(),
                                                new ApplicationCertificateMock(),
                                                new MockMeteringClient(),
-                                               new MemoryGlobalRoutingService());
+                                               globalRoutingService);
         // Calculate initial versions
         controller.updateVersionStatus(VersionStatus.compute(controller));
         return controller;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index ccafab622d0..60ac01bdf57 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -6,13 +6,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.rotation.RotationState;
-import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
-import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -90,48 +84,6 @@ public class DeploymentMetricsMaintainerTest {
         assertEquals(5, deployment.get().activity().lastWritesPerSecond().getAsDouble(), Double.MIN_VALUE);
     }
 
-    @Test
-    public void updates_rotation_status() {
-        DeploymentTester tester = new DeploymentTester();
-        MetricsServiceMock metricsService = tester.controllerTester().metricsService();
-        DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
-        Application application = tester.createApplication("app1", "tenant1", 1, 1L);
-        ZoneId zone1 = ZoneId.from("prod", "us-west-1");
-        ZoneId zone2 = ZoneId.from("prod", "us-east-3");
-
-        // Deploy application with global rotation
-        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
-                .environment(Environment.prod)
-                .globalServiceId("foo")
-                .region(zone1.region().value())
-                .region(zone2.region().value())
-                .build();
-        tester.deployCompletely(application, applicationPackage);
-
-        Supplier<Application> app = () -> tester.application(application.id());
-        Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1);
-        Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2);
-        String assignedRotation = "rotation-fqdn-01";
-        tester.controllerTester().metricsService().addRotation(assignedRotation);
-
-        // No status gathered yet
-        assertEquals(RotationState.unknown, app.get().rotationStatus().of(deployment1.get()));
-        assertEquals(RotationState.unknown, app.get().rotationStatus().of(deployment2.get()));
-
-        // One rotation out, one in
-        metricsService.setZoneIn(assignedRotation, "proxy.prod.us-west-1.vip.test");
-        metricsService.setZoneOut(assignedRotation,"proxy.prod.us-east-3.vip.test");
-        maintainer.maintain();
-        assertEquals(RotationState.in, app.get().rotationStatus().of(deployment1.get()));
-        assertEquals(RotationState.out, app.get().rotationStatus().of(deployment2.get()));
-
-        // All rotations in
-        metricsService.setZoneIn(assignedRotation,"proxy.prod.us-east-3.vip.test");
-        maintainer.maintain();
-        assertEquals(RotationState.in, app.get().rotationStatus().of(deployment1.get()));
-        assertEquals(RotationState.in, app.get().rotationStatus().of(deployment2.get()));
-    }
-
     private static DeploymentMetricsMaintainer maintainer(Controller controller) {
         return new DeploymentMetricsMaintainer(controller, Duration.ofDays(1), new JobControl(controller.curator()));
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java
new file mode 100644
index 00000000000..4ef77573f93
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java
@@ -0,0 +1,104 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.rotation.RotationState;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class RotationStatusUpdaterTest {
+
+    @Test
+    public void updates_rotation_status() {
+        var tester = new DeploymentTester();
+        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true);
+        var globalRotationService = tester.controllerTester().globalRoutingService();
+        var updater = new RotationStatusUpdater(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()));
+
+        var application = tester.createApplication("app1", "tenant1", 1, 1L);
+        var zone1 = ZoneId.from("prod", "us-west-1");
+        var zone2 = ZoneId.from("prod", "us-east-3");
+        var zone3 = ZoneId.from("prod", "eu-west-1");
+
+        // Deploy application with global rotation
+        var applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .globalServiceId("foo")
+                .region(zone1.region().value())
+                .region(zone2.region().value())
+                .build();
+        tester.deployCompletely(application, applicationPackage);
+
+        Supplier<Application> app = () -> tester.application(application.id());
+        Supplier<Deployment> deployment1 = () -> app.get().deployments().get(zone1);
+        Supplier<Deployment> deployment2 = () -> app.get().deployments().get(zone2);
+        Supplier<Deployment> deployment3 = () -> app.get().deployments().get(zone3);
+
+        // No status gathered yet
+        var rotation1 = app.get().rotations().get(0).rotationId();
+        assertEquals(RotationState.unknown, app.get().rotationStatus().of(rotation1, deployment1.get()));
+        assertEquals(RotationState.unknown, app.get().rotationStatus().of(rotation1, deployment2.get()));
+
+        // First rotation: One zone out, one in
+        var rotationName1 = "rotation-fqdn-01";
+        globalRotationService.setStatus(rotationName1, zone1, RotationStatus.IN)
+                             .setStatus(rotationName1, zone2, RotationStatus.OUT);
+        updater.maintain();
+        assertEquals(RotationState.in, app.get().rotationStatus().of(rotation1, deployment1.get()));
+        assertEquals(RotationState.out, app.get().rotationStatus().of(rotation1, deployment2.get()));
+
+        // First rotation: All zones in
+        globalRotationService.setStatus(rotationName1, zone2, RotationStatus.IN);
+        updater.maintain();
+        assertEquals(RotationState.in, app.get().rotationStatus().of(rotation1, deployment1.get()));
+        assertEquals(RotationState.in, app.get().rotationStatus().of(rotation1, deployment2.get()));
+
+        // Another rotation is assigned
+        applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .region(zone1.region().value())
+                .region(zone2.region().value())
+                .region(zone3.region().value())
+                .endpoint("default", "default", "us-east-3", "us-west-1")
+                .endpoint("eu", "default", "eu-west-1")
+                .build();
+        tester.deployCompletely(application, applicationPackage, BuildJob.defaultBuildNumber + 1);
+        assertEquals(2, app.get().rotations().size());
+
+        // Second rotation: No status gathered yet
+        var rotation2 = app.get().rotations().get(1).rotationId();
+        updater.maintain();
+        assertEquals(RotationState.unknown, app.get().rotationStatus().of(rotation2, deployment3.get()));
+
+        // Status of third zone is retrieved via second rotation
+        var rotationName2 = "rotation-fqdn-02";
+        globalRotationService.setStatus(rotationName2, zone3, RotationStatus.IN);
+        updater.maintain();
+        assertEquals(RotationState.in, app.get().rotationStatus().of(rotation2, deployment3.get()));
+
+        // Each rotation only has status for their configured zones
+        assertEquals("Rotation " + rotation1 + " does not know about " + deployment3.get(), RotationState.unknown,
+                     app.get().rotationStatus().of(rotation1, deployment3.get()));
+        assertEquals("Rotation " + rotation2 + " does not know about " + deployment1.get(), RotationState.unknown,
+                     app.get().rotationStatus().of(rotation2, deployment1.get()));
+        assertEquals("Rotation " + rotation2 + " does not know about " + deployment2.get(), RotationState.unknown,
+                     app.get().rotationStatus().of(rotation2, deployment2.get()));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 9bc5c879961..e93513ec088 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -1677,16 +1677,16 @@ public class ApplicationApiTest extends ControllerContainerTest {
         List<Application> applicationList = applicationController.asList();
         applicationList.forEach(application -> {
                 applicationController.lockIfPresent(application.id(), locked ->
-                        applicationController.store(locked.withRotationStatus(rotationStatus(application))));
+                        applicationController.store(locked.with(rotationStatus(application))));
         });
     }
 
     private RotationStatus rotationStatus(Application application) {
         return controllerTester.controller().applications().rotationRepository().getRotation(application)
                 .map(rotation -> {
-                    var rotationStatus = controllerTester.controller().metricsService().getRotationStatus(rotation.name());
+                    var rotationStatus = controllerTester.controller().globalRoutingService().getHealthStatus(rotation.name());
                     var statusMap = new LinkedHashMap<ZoneId, RotationState>();
-                    rotationStatus.forEach((hostname, status) -> statusMap.put(ZoneId.from("prod", hostname.value()), RotationState.in));
+                    rotationStatus.forEach((zone, status) -> statusMap.put(zone, RotationState.in));
                     return new RotationStatus(Map.of(rotation.id(), statusMap));
                 })
                 .orElse(RotationStatus.EMPTY);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 1b5b679cb8a..c8ab8a92583 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -55,6 +55,9 @@
       "name": "ResourceMeterMaintainer"
     },
     {
+      "name": "RotationStatusUpdater"
+    },
+    {
       "name": "SystemUpgrader"
     },
     {