Merge pull request #20336 from vespa-engine/mpolden/change-all-upstreams

Change routing status for all known clusters

7 files changed, 80 insertions, 27 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 943d6ac7b18..c58bb0e5fab 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -74,6 +74,7 @@ public class RoutingController {
     private final RoutingPolicies routingPolicies;
     private final RotationRepository rotationRepository;
     private final BooleanFlag hideSharedRoutingEndpoint;
+    private final BooleanFlag changeRoutingStatusOfAllUpstreams;
 
     public RoutingController(Controller controller, RotationsConfig rotationsConfig) {
         this.controller = Objects.requireNonNull(controller, "controller must be non-null");
@@ -82,6 +83,7 @@ public class RoutingController {
                                                          controller.applications(),
                                                          controller.curator());
         this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource());
+        this.changeRoutingStatusOfAllUpstreams = Flags.CHANGE_ROUTING_STATUS_OF_ALL_UPSTREAMS.bindTo(controller.flagSource());
     }
 
     /** Create a routing context for given deployment */
@@ -90,7 +92,8 @@ public class RoutingController {
             return new SharedDeploymentRoutingContext(deployment,
                                                       this,
                                                       controller.serviceRegistry().configServer(),
-                                                      controller.clock());
+                                                      controller.clock(),
+                                                      changeRoutingStatusOfAllUpstreams.value());
         }
         return new ExclusiveDeploymentRoutingContext(deployment, this);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 544822e3be3..8f37d287c1a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -133,10 +133,10 @@ public class Endpoint {
         return routingMethod.isShared() && scope == Scope.global;
     }
 
-    /** Returns the upstream ID of given deployment. This *must* match what the routing layer generates */
-    public String upstreamIdOf(DeploymentId deployment) {
+    /** Returns the upstream name of given deployment. This *must* match what the routing layer generates */
+    public String upstreamName(DeploymentId deployment) {
         if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name");
-        return upstreamIdOf(cluster.value(), deployment.applicationId(), deployment.zoneId());
+        return upstreamName(cluster.value(), deployment.applicationId(), deployment.zoneId());
     }
 
     @Override
@@ -269,7 +269,7 @@ public class Endpoint {
         return suffix;
     }
 
-    private static String upstreamIdOf(String name, ApplicationId application, ZoneId zone) {
+    private static String upstreamName(String name, ApplicationId application, ZoneId zone) {
         return Stream.of(namePart(name, ""),
                          instancePart(Optional.of(application.instance()), ""),
                          application.application().value(),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index ed16c86c94e..cf85d862041 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1555,7 +1555,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
         if (primaryEndpoint.isPresent()) {
             DeploymentRoutingContext context = controller.routing().of(deploymentId);
             RoutingStatus status = context.routingStatus();
-            array.addString(primaryEndpoint.get().upstreamIdOf(deploymentId));
+            array.addString(primaryEndpoint.get().upstreamName(deploymentId));
             Cursor statusObject = array.addObject();
             statusObject.setString("status", status.value().name());
             statusObject.setString("reason", "");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
index 6fd8a3a84d5..e784666f22a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
@@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
 import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 
 import java.time.Clock;
-import java.time.Instant;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -71,48 +71,99 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
 
         private final Clock clock;
         private final ConfigServer configServer;
+        private final boolean changeAllUpstreams;
 
-        public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) {
+        public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock, boolean changeAllUpstreams) {
             super(deployment, RoutingMethod.shared, controller);
             this.clock = Objects.requireNonNull(clock);
             this.configServer = Objects.requireNonNull(configServer);
+            this.changeAllUpstreams = changeAllUpstreams;
         }
 
         @Override
         public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
+            if (!changeAllUpstreams) {
+                setLegacyRoutingStatus(value, agent);
+                return;
+            }
+            EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in
+                                                                  ? EndpointStatus.Status.in
+                                                                  : EndpointStatus.Status.out,
+                                                          agent.name(),
+                                                          clock.instant());
+            try {
+                configServer.setGlobalRotationStatus(deployment, upstreamNames(), newStatus);
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to change rotation status of " + deployment, e);
+            }
+        }
+
+        @Override
+        public RoutingStatus routingStatus() {
+            if (!changeAllUpstreams) {
+                return legacyRoutingStatus();
+            }
+
+            // In a given deployment, all upstreams (clusters) share the same status, so we can query using any
+            // upstream name
+            String upstreamName = upstreamNames().get(0);
+            EndpointStatus status = configServer.getGlobalRotationStatus(deployment, upstreamName);
+            RoutingStatus.Agent agent;
+            try {
+                agent = RoutingStatus.Agent.valueOf(status.agent().toLowerCase());
+            } catch (IllegalArgumentException e) {
+                agent = RoutingStatus.Agent.unknown;
+            }
+            return new RoutingStatus(status.status() == EndpointStatus.Status.in
+                                             ? RoutingStatus.Value.in
+                                             : RoutingStatus.Value.out,
+                                     agent,
+                                     status.changedAt());
+        }
+
+        private List<String> upstreamNames() {
+            List<String> upstreamNames = controller.readEndpointsOf(deployment)
+                                                   .scope(Endpoint.Scope.zone)
+                                                   .shared()
+                                                   .mapToList(endpoint -> endpoint.upstreamName(deployment));
+            if (upstreamNames.isEmpty()) {
+                throw new IllegalArgumentException("No upstream names found for " + deployment);
+            }
+            return upstreamNames;
+        }
+
+        private void setLegacyRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
             EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in
                                                                   ? EndpointStatus.Status.in
                                                                   : EndpointStatus.Status.out,
-                                                          "",
                                                           agent.name(),
-                                                          clock.instant().getEpochSecond());
+                                                          clock.instant());
             primaryEndpoint().ifPresent(endpoint -> {
                 try {
-                    configServer.setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), newStatus);
+                    configServer.setGlobalRotationStatus(deployment, List.of(endpoint.upstreamName(deployment)), newStatus);
                 } catch (Exception e) {
                     throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e);
                 }
             });
         }
 
-        @Override
-        public RoutingStatus routingStatus() {
+        private RoutingStatus legacyRoutingStatus() {
             Optional<EndpointStatus> status = primaryEndpoint().map(endpoint -> {
-                var upstreamName = endpoint.upstreamIdOf(deployment);
+                var upstreamName = endpoint.upstreamName(deployment);
                 return configServer.getGlobalRotationStatus(deployment, upstreamName);
             });
             if (status.isEmpty()) return RoutingStatus.DEFAULT;
             RoutingStatus.Agent agent;
             try {
-                agent = RoutingStatus.Agent.valueOf(status.get().getAgent().toLowerCase());
+                agent = RoutingStatus.Agent.valueOf(status.get().agent().toLowerCase());
             } catch (IllegalArgumentException e) {
                 agent = RoutingStatus.Agent.unknown;
             }
-            return new RoutingStatus(status.get().getStatus() == EndpointStatus.Status.in
+            return new RoutingStatus(status.get().status() == EndpointStatus.Status.in
                                              ? RoutingStatus.Value.in
                                              : RoutingStatus.Value.out,
                                      agent,
-                                     Instant.ofEpochSecond(status.get().getEpoch()));
+                                     status.get().changedAt());
         }
 
         private Optional<Endpoint> primaryEndpoint() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
index 46d27911de4..e50c32d0e5d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
@@ -387,8 +387,8 @@ public class EndpointTest {
                 "c2.i2.a2.t2.us-north-1.prod",
                 Endpoint.of(instance2).target(EndpointId.of("ignored2"), ClusterSpec.Id.from("c2"), List.of(zone2)).on(Port.tls(4443)).in(SystemName.main)
         );
-        tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(zone)));
-        tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(zone2)));
+        tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamName(zone)));
+        tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamName(zone2)));
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index f2fc624630c..aa53d09be04 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -80,7 +80,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
     private final Map<DeploymentId, Application> applications = new LinkedHashMap<>();
     private final Set<ZoneId> inactiveZones = new HashSet<>();
-    private final Map<String, EndpointStatus> endpoints = new HashMap<>();
+    private final Map<DeploymentId, EndpointStatus> endpoints = new HashMap<>();
     private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock();
     private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>();
     private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>();
@@ -536,8 +536,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) {
-        endpoints.put(upstreamName, status);
+    public void setGlobalRotationStatus(DeploymentId deployment, List<String> upstreamNames, EndpointStatus status) {
+        endpoints.put(deployment, status);
     }
 
     @Override
@@ -550,9 +550,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) {
-        EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757L);
-        return endpoints.getOrDefault(endpoint, result);
+    public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String upstreamName) {
+        EndpointStatus status = new EndpointStatus(EndpointStatus.Status.in, "", Instant.ofEpochSecond(1497618757L));
+        return endpoints.getOrDefault(deployment, status);
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 6e8445102c3..6cf3e89bdfe 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -52,7 +52,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
-import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
@@ -1877,7 +1876,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         RoutingStatus status = context.routingStatus();
         assertEquals(value, status.value());
         assertEquals(agent, status.agent());
-        assertEquals(changedAt.truncatedTo(ChronoUnit.SECONDS), status.changedAt());
+        assertEquals(changedAt, status.changedAt());
     }
 
     private static class RequestBuilder implements Supplier<Request> {