Fix global rotation override

Removes guesswork and sends the correct identifier (upstream name) to the API on the config server.

10 files changed, 140 insertions, 151 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index eb86f0c2919..7214a371460 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstance
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
@@ -40,7 +41,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.JobList;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
@@ -60,16 +60,14 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -180,83 +178,35 @@ public class ApplicationController {
                       .orElse(controller.systemVersion());
     }
 
-    /**
-     * Set the rotations marked as 'global' either 'in' or 'out of' service.
-     *
-     * @return The canonical endpoint altered if any
-     * @throws IOException if rotation status cannot be updated
-     */
-    public List<String> setGlobalRotationStatus(DeploymentId deploymentId, EndpointStatus status) throws IOException {
-        List<String> rotations = new ArrayList<>();
-        Optional<String> endpoint = getCanonicalGlobalEndpoint(deploymentId);
-
-        if (endpoint.isPresent()) {
-            configServer.setGlobalRotationStatus(deploymentId, endpoint.get(), status);
-            rotations.add(endpoint.get());
-        }
-
-        return rotations;
-    }
-
-    /**
-     * Get the endpoint status for the global endpoint of this application
-     *
-     * @return Map between the endpoint and the rotation status
-     * @throws IOException if global rotation status cannot be determined
-     */
-    public Map<String, EndpointStatus> getGlobalRotationStatus(DeploymentId deploymentId) throws IOException {
-        Map<String, EndpointStatus> result = new HashMap<>();
-        Optional<String> endpoint = getCanonicalGlobalEndpoint(deploymentId);
-
-        if (endpoint.isPresent()) {
-            EndpointStatus status = configServer.getGlobalRotationStatus(deploymentId, endpoint.get());
-            result.put(endpoint.get(), status);
-        }
-
-        return result;
+    /** Change the global endpoint status for given deployment */
+    public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) {
+        findGlobalEndpoint(deployment).map(endpoint -> {
+            try {
+                configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(), status);
+                return endpoint;
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to set rotation status of " + deployment, e);
+            }
+        }).orElseThrow(() -> new IllegalArgumentException("No global endpoint exists for " + deployment));
     }
 
-    /**
-     * Global rotations (plural as we can have aliases) map to exactly one service endpoint.
-     * This method finds that one service endpoint and strips the URI part that
-     * the routingGenerator is wrapping around the endpoint.
-     *
-     * @param deploymentId The deployment to retrieve global service endpoint for
-     * @return Empty if no global endpoint exist, otherwise the service endpoint ([clustername.]app.tenant.region.env)
-     */
-    Optional<String> getCanonicalGlobalEndpoint(DeploymentId deploymentId) throws IOException {
-        Map<String, RoutingEndpoint> hostToGlobalEndpoint = new HashMap<>();
-        Map<String, String> hostToCanonicalEndpoint = new HashMap<>();
-
-        for (RoutingEndpoint endpoint : routingGenerator.endpoints(deploymentId)) {
+    /** Get global endpoint status for given deployment */
+    public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) {
+        return findGlobalEndpoint(deployment).map(endpoint -> {
             try {
-                URI uri = new URI(endpoint.getEndpoint());
-                String serviceEndpoint = uri.getHost();
-                if (serviceEndpoint == null) {
-                    throw new IOException("Unexpected endpoints returned from the Routing Generator");
-                }
-                String canonicalEndpoint = serviceEndpoint
-                        .replaceAll(".vespa.yahooapis.com", "")
-                        .replaceAll(".vespa.oath.cloud", "");
-                String hostname = endpoint.getHostname();
-
-                // Book-keeping
-                if (endpoint.isGlobal()) {
-                    hostToGlobalEndpoint.put(hostname, endpoint);
-                } else {
-                    hostToCanonicalEndpoint.put(hostname, canonicalEndpoint);
-                }
-
-                // Return as soon as we have a map between a global and a canonical endpoint
-                if (hostToGlobalEndpoint.containsKey(hostname) && hostToCanonicalEndpoint.containsKey(hostname)) {
-                    return Optional.of(hostToCanonicalEndpoint.get(hostname));
-                }
-            } catch (URISyntaxException use) {
-                throw new IOException(use);
+                EndpointStatus status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName());
+                return Collections.singletonMap(endpoint, status);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to get rotation status of " + deployment, e);
             }
-        }
+        }).orElseGet(Collections::emptyMap);
+    }
 
-        return Optional.empty();
+    /** Find the global endpoint of given deployment, if any */
+    public Optional<RoutingEndpoint> findGlobalEndpoint(DeploymentId deployment) {
+        return routingGenerator.endpoints(deployment).stream()
+                               .filter(RoutingEndpoint::isGlobal)
+                               .findFirst();
     }
 
     /**
@@ -553,7 +503,7 @@ public class ApplicationController {
 
         try {
             return Optional.of(ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
-                                                                    .map(RoutingEndpoint::getEndpoint)
+                                                                    .map(RoutingEndpoint::endpoint)
                                                                     .map(URI::create)
                                                                     .iterator()));
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index cfd36a25de8..57694517c3b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -53,6 +53,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
@@ -590,31 +591,26 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) {
-
-        // Check if request is authorized
-        Optional<Tenant> existingTenant = controller.tenants().tenant(tenantName);
-        if (!existingTenant.isPresent())
-            return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");
-
-
-        // Decode payload (reason) and construct parameter to the configserver
+        Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName));
+        ZoneId zone = ZoneId.from(environment, region);
+        Deployment deployment = application.deployments().get(zone);
+        if (deployment == null) {
+            throw new NotExistsException(application + " has no deployment in " + zone);
+        }
 
         Inspector requestData = toSlime(request.getData()).get();
         String reason = mandatory("reason", requestData).asString();
         String agent = getUserPrincipal(request).getIdentity().getFullName();
         long timestamp = controller.clock().instant().getEpochSecond();
         EndpointStatus.Status status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out;
-        EndpointStatus endPointStatus = new EndpointStatus(status, reason, agent, timestamp);
-
-        // DeploymentId identifies the zone and application we are dealing with
-        DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
-                                                     ZoneId.from(environment, region));
-        try {
-            List<String> rotations = controller.applications().setGlobalRotationStatus(deploymentId, endPointStatus);
-            return new MessageResponse(String.format("Rotations %s successfully set to %s service", rotations.toString(), inService ? "in" : "out of"));
-        } catch (IOException e) {
-            return ErrorResponse.internalServerError("Unable to alter rotation status: " + e.getMessage());
-        }
+        EndpointStatus endpointStatus = new EndpointStatus(status, reason, agent, timestamp);
+        controller.applications().setGlobalRotationStatus(new DeploymentId(application.id(), deployment.zone()),
+                                                          endpointStatus);
+        return new MessageResponse(String.format("Successfully set %s in %s.%s %s service",
+                                                 application.id().toShortString(),
+                                                 deployment.zone().environment().value(),
+                                                 deployment.zone().region().value(),
+                                                 inService ? "in" : "out of"));
     }
 
     private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) {
@@ -623,20 +619,16 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                      ZoneId.from(environment, region));
 
         Slime slime = new Slime();
-        Cursor c1 = slime.setObject().setArray("globalrotationoverride");
-        try {
-            Map<String, EndpointStatus> rotations = controller.applications().getGlobalRotationStatus(deploymentId);
-            for (String rotation : rotations.keySet()) {
-                EndpointStatus currentStatus = rotations.get(rotation);
-                c1.addString(rotation);
-                Cursor c2 = c1.addObject();
-                c2.setString("status", currentStatus.getStatus().name());
-                c2.setString("reason", currentStatus.getReason() == null ? "" : currentStatus.getReason());
-                c2.setString("agent", currentStatus.getAgent()  == null ? "" : currentStatus.getAgent());
-                c2.setLong("timestamp", currentStatus.getEpoch());
-            }
-        } catch (IOException e) {
-            return ErrorResponse.internalServerError("Unable to get rotation status: " + e.getMessage());
+        Cursor array = slime.setObject().setArray("globalrotationoverride");
+        Map<RoutingEndpoint, EndpointStatus> status = controller.applications().globalRotationStatus(deploymentId);
+        for (RoutingEndpoint endpoint : status.keySet()) {
+            EndpointStatus currentStatus = status.get(endpoint);
+            array.addString(endpoint.upstreamName());
+            Cursor statusObject = array.addObject();
+            statusObject.setString("status", currentStatus.getStatus().name());
+            statusObject.setString("reason", currentStatus.getReason() == null ? "" : currentStatus.getReason());
+            statusObject.setString("agent", currentStatus.getAgent() == null ? "" : currentStatus.getAgent());
+            statusObject.setLong("timestamp", currentStatus.getEpoch());
         }
 
         return new SlimeJsonResponse(slime);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 25a051e192f..b7a881c672e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -15,15 +15,16 @@ import com.yahoo.vespa.athenz.api.OktaAccessToken;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -31,16 +32,17 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.time.Duration;
-import java.util.List;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static com.yahoo.config.provision.SystemName.main;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
 import static java.time.temporal.ChronoUnit.MILLIS;
@@ -218,29 +220,51 @@ public class ControllerTest {
     }
 
     @Test
-    public void testGlobalRotations() throws IOException {
-        // Setup tester and app def
+    public void testGlobalRotations() {
+        // Setup
         ControllerTester tester = new ControllerTester();
         ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName());
-        ApplicationId appId = ApplicationId.from("tenant", "app1", "default");
-        DeploymentId deployId = new DeploymentId(appId, zone);
+        ApplicationId app = ApplicationId.from("tenant", "app1", "default");
+        DeploymentId deployment = new DeploymentId(app, zone);
+        tester.routingGenerator().putEndpoints(deployment, Arrays.asList(
+                new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"),
+                new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
+                new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3"),
+                new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"),
+                new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1")
+        ));
+
+        Supplier<Map<RoutingEndpoint, EndpointStatus>> rotationStatus = () -> tester.controller().applications().globalRotationStatus(deployment);
+        Function<String, Optional<EndpointStatus>> findStatusByUpstream = (upstreamName) -> {
+            return rotationStatus.get()
+                                 .entrySet().stream()
+                                 .filter(kv -> kv.getKey().upstreamName().equals(upstreamName))
+                                 .findFirst()
+                                 .map(Map.Entry::getValue);
+        };
 
         // Check initial rotation status
-        Map<String, EndpointStatus> rotationStatus = tester.controller().applications().getGlobalRotationStatus(deployId);
-        assertEquals(1, rotationStatus.size());
-
-        assertEquals(rotationStatus.get("qrs-endpoint").getStatus(), EndpointStatus.Status.in);
+        assertEquals(1, rotationStatus.get().size());
+        assertEquals(findStatusByUpstream.apply("upstream1").get().getStatus(), EndpointStatus.Status.in);
 
         // Set the global rotations out of service
         EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "unit-test", "Test", tester.clock().instant().getEpochSecond());
-        List<String> overrides = tester.controller().applications().setGlobalRotationStatus(deployId, status);
-        assertEquals(1, overrides.size());
-
-        // Recheck the override rotation status
-        rotationStatus = tester.controller().applications().getGlobalRotationStatus(deployId);
-        assertEquals(1, rotationStatus.size());
-        assertEquals(rotationStatus.get("qrs-endpoint").getStatus(), EndpointStatus.Status.out);
-        assertEquals("unit-test", rotationStatus.get("qrs-endpoint").getReason());
+        tester.controller().applications().setGlobalRotationStatus(deployment, status);
+        assertEquals(1, rotationStatus.get().size());
+        assertEquals(findStatusByUpstream.apply("upstream1").get().getStatus(), EndpointStatus.Status.out);
+        assertEquals("unit-test", findStatusByUpstream.apply("upstream1").get().getReason());
+
+        // Deployment without a global endpoint
+        tester.routingGenerator().putEndpoints(deployment, Arrays.asList(
+                new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"),
+                new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
+                new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3")
+        ));
+        assertFalse("No global endpoint exists", findStatusByUpstream.apply("upstream1").isPresent());
+        try {
+            tester.controller().applications().setGlobalRotationStatus(deployment, status);
+            fail("Expected exception");
+        } catch (IllegalArgumentException ignored) {}
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
index ebda3482a50..80cbbfa2c30 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
@@ -107,7 +107,12 @@ public class InternalDeploymentTester {
                                                                                          zone.region().value(),
                                                                                          zone.environment().value()),
                                                                            "host1",
-                                                                           false)));
+                                                                           false,
+                                                                           String.format("cluster1.%s.%s.%s.%s",
+                                                                                         id.application().value(),
+                                                                                         id.tenant().value(),
+                                                                                         zone.region().value(),
+                                                                                         zone.environment().value()))));
     }
 
     /**
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index bb4b4fae5a5..c5484e1db07 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import org.junit.Before;
@@ -32,7 +31,6 @@ import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.debug;
@@ -236,7 +234,7 @@ public class InternalStepRunnerTest {
         RunId id = tester.startSystemTestTests();
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.endTests));
-        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()))).get(0).getEndpoint()),
+        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system()))).get(0).endpoint()),
                      tester.cloud().testerUrl());
         Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get();
         assertEquals(appId.serializedForm(), configObject.field("application").asString());
@@ -245,7 +243,7 @@ public class InternalStepRunnerTest {
         assertEquals(1, configObject.field("endpoints").children());
         assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).entries());
         configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).traverse((ArrayTraverser) (__, endpoint) ->
-                assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(tester.tester().controller().system()))).get(0).getEndpoint(), endpoint.asString()));
+                assertEquals(tester.routing().endpoints(new DeploymentId(appId, JobType.systemTest.zone(tester.tester().controller().system()))).get(0).endpoint(), endpoint.asString()));
 
         long lastId = tester.jobs().details(id).get().lastId().getAsLong();
         tester.cloud().add(new LogEntry(0, 123, info, "Ready!"));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
index 28b255e9d0a..7c32e593eec 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
@@ -19,27 +19,28 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class RoutingGeneratorMock implements RoutingGenerator {
 
-    private final Map<DeploymentId, List<RoutingEndpoint>> allEndpoints = new ConcurrentHashMap<>();
+    private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>();
+
     private static final List<RoutingEndpoint> defaultEndpoints =
-            Arrays.asList(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false),
-                          new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false),
-                          new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false),
-                          new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true),
-                          new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true));
+            Arrays.asList(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream3"),
+                          new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
+                          new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream2"),
+                          new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"),
+                          new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"));
 
     @Override
     public List<RoutingEndpoint> endpoints(DeploymentId deployment) {
-        return allEndpoints.isEmpty()
+        return routingTable.isEmpty()
                 ? defaultEndpoints
-                : allEndpoints.getOrDefault(deployment, Collections.emptyList());
+                : routingTable.getOrDefault(deployment, Collections.emptyList());
     }
 
     public void putEndpoints(DeploymentId deployment, List<RoutingEndpoint> endpoints) {
-        allEndpoints.put(deployment, endpoints);
+        routingTable.put(deployment, endpoints);
     }
 
     public void removeEndpoints(DeploymentId deployment) {
-        allEndpoints.remove(deployment);
+        routingTable.remove(deployment);
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 22ef61839be..981972356a5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -556,6 +556,25 @@ public class ApplicationApiTest extends ControllerContainerTest {
                         .submit();
         setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1"));
 
+        // Invalid application fails
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation", GET)
+                                      .userIdentity(USER_ID),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"tenant2.application2 not found\"}",
+                              400);
+
+        // Invalid deployment fails
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation", GET)
+                                      .userIdentity(USER_ID),
+                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in zone prod.us-east-3 in default\"}",
+                              404);
+
+        // Change status of non-existing deployment fails
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation/override", PUT)
+                                      .userIdentity(USER_ID)
+                                      .data("{\"reason\":\"unit-test\"}"),
+                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in zone prod.us-east-3 in default\"}",
+                              404);
+
         // GET global rotation status
         setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1"));
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", GET)
@@ -570,13 +589,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // SET global rotation override status
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", PUT)
                                       .userIdentity(USER_ID)
-                                      .data("{\"reason\":\"because i can\"}"),
+                                      .data("{\"reason\":\"unit-test\"}"),
                               new File("global-rotation-put.json"));
 
         // DELETE global rotation override status
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", DELETE)
                                       .userIdentity(USER_ID)
-                                      .data("{\"reason\":\"because i can\"}"),
+                                      .data("{\"reason\":\"unit-test\"}"),
                               new File("global-rotation-delete.json"));
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json
index c7ee62a3fcc..048ddcbc5c5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json
@@ -1 +1 @@
-{"message":"Rotations [qrs-endpoint] successfully set to in service"}
\ No newline at end of file
+{"message":"Successfully set tenant1.application1 in prod.us-west-1 in service"}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
index 5fe96bd4d78..860d6c49cdf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
@@ -1 +1 @@
-{"globalrotationoverride":["qrs-endpoint",{"status":"in","reason":"","agent":"","timestamp":1497618757}]}
\ No newline at end of file
+{"globalrotationoverride":["upstream1",{"status":"in","reason":"","agent":"","timestamp":1497618757}]}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json
index abe9658d882..f67b8dd56d9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json
@@ -1 +1 @@
-{"message":"Rotations [qrs-endpoint] successfully set to out of service"}
\ No newline at end of file
+{"message":"Successfully set tenant1.application1 in prod.us-west-1 out of service"}