Change routing status for all known clusters

3 files changed, 67 insertions, 10 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 943d6ac7b18..c58bb0e5fab 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -74,6 +74,7 @@ public class RoutingController {
     private final RoutingPolicies routingPolicies;
     private final RotationRepository rotationRepository;
     private final BooleanFlag hideSharedRoutingEndpoint;
+    private final BooleanFlag changeRoutingStatusOfAllUpstreams;
 
     public RoutingController(Controller controller, RotationsConfig rotationsConfig) {
         this.controller = Objects.requireNonNull(controller, "controller must be non-null");
@@ -82,6 +83,7 @@ public class RoutingController {
                                                          controller.applications(),
                                                          controller.curator());
         this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource());
+        this.changeRoutingStatusOfAllUpstreams = Flags.CHANGE_ROUTING_STATUS_OF_ALL_UPSTREAMS.bindTo(controller.flagSource());
     }
 
     /** Create a routing context for given deployment */
@@ -90,7 +92,8 @@ public class RoutingController {
             return new SharedDeploymentRoutingContext(deployment,
                                                       this,
                                                       controller.serviceRegistry().configServer(),
-                                                      controller.clock());
+                                                      controller.clock(),
+                                                      changeRoutingStatusOfAllUpstreams.value());
         }
         return new ExclusiveDeploymentRoutingContext(deployment, this);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
index 569c767d302..b61548c7e84 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 
 import java.time.Clock;
 import java.time.Instant;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -71,15 +72,69 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
 
         private final Clock clock;
         private final ConfigServer configServer;
+        private final boolean changeAllUpstreams;
 
-        public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) {
+        public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock, boolean changeAllUpstreams) {
             super(deployment, RoutingMethod.shared, controller);
             this.clock = Objects.requireNonNull(clock);
             this.configServer = Objects.requireNonNull(configServer);
+            this.changeAllUpstreams = changeAllUpstreams;
         }
 
         @Override
         public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
+            if (!changeAllUpstreams) {
+                setLegacyRoutingStatus(value, agent);
+                return;
+            }
+            EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in
+                                                                  ? EndpointStatus.Status.in
+                                                                  : EndpointStatus.Status.out,
+                                                          "",
+                                                          agent.name(),
+                                                          clock.instant().getEpochSecond());
+            try {
+                configServer.setGlobalRotationStatus(deployment, upstreamNames(), newStatus);
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to change rotation status of " + deployment, e);
+            }
+        }
+
+        @Override
+        public RoutingStatus routingStatus() {
+            if (!changeAllUpstreams) {
+                return legacyRoutingStatus();
+            }
+
+            // In a given deployment, all upstreams (clusters) share the same status, so we can query using any
+            // upstream name
+            String upstreamName = upstreamNames().get(0);
+            EndpointStatus status = configServer.getGlobalRotationStatus(deployment, upstreamName);
+            RoutingStatus.Agent agent;
+            try {
+                agent = RoutingStatus.Agent.valueOf(status.getAgent().toLowerCase());
+            } catch (IllegalArgumentException e) {
+                agent = RoutingStatus.Agent.unknown;
+            }
+            return new RoutingStatus(status.getStatus() == EndpointStatus.Status.in
+                                             ? RoutingStatus.Value.in
+                                             : RoutingStatus.Value.out,
+                                     agent,
+                                     Instant.ofEpochSecond(status.getEpoch()));
+        }
+
+        private List<String> upstreamNames() {
+            List<String> upstreamNames = controller.readEndpointsOf(deployment)
+                                                   .scope(Endpoint.Scope.zone)
+                                                   .shared()
+                                                   .mapToList(endpoint -> endpoint.upstreamName(deployment));
+            if (upstreamNames.isEmpty()) {
+                throw new IllegalArgumentException("No upstream names found for " + deployment);
+            }
+            return upstreamNames;
+        }
+
+        private void setLegacyRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
             EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in
                                                                   ? EndpointStatus.Status.in
                                                                   : EndpointStatus.Status.out,
@@ -88,15 +143,14 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
                                                           clock.instant().getEpochSecond());
             primaryEndpoint().ifPresent(endpoint -> {
                 try {
-                    configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(deployment), newStatus);
+                    configServer.setGlobalRotationStatus(deployment, List.of(endpoint.upstreamName(deployment)), newStatus);
                 } catch (Exception e) {
                     throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e);
                 }
             });
         }
 
-        @Override
-        public RoutingStatus routingStatus() {
+        private RoutingStatus legacyRoutingStatus() {
             Optional<EndpointStatus> status = primaryEndpoint().map(endpoint -> {
                 var upstreamName = endpoint.upstreamName(deployment);
                 return configServer.getGlobalRotationStatus(deployment, upstreamName);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index f2fc624630c..8163618fc1e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -80,7 +80,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
     private final Map<DeploymentId, Application> applications = new LinkedHashMap<>();
     private final Set<ZoneId> inactiveZones = new HashSet<>();
-    private final Map<String, EndpointStatus> endpoints = new HashMap<>();
+    private final Map<DeploymentId, EndpointStatus> endpoints = new HashMap<>();
     private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock();
     private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>();
     private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>();
@@ -536,8 +536,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) {
-        endpoints.put(upstreamName, status);
+    public void setGlobalRotationStatus(DeploymentId deployment, List<String> upstreamNames, EndpointStatus status) {
+        endpoints.put(deployment, status);
     }
 
     @Override
@@ -550,9 +550,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) {
+    public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String upstreamName) {
         EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757L);
-        return endpoints.getOrDefault(endpoint, result);
+        return endpoints.getOrDefault(deployment, result);
     }
 
     @Override