Wire reindexing status through controller and application v4

4 files changed, 166 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index e275b4e9dfd..9854d884bd3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -35,6 +35,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.aws.ApplicationRoles;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
@@ -173,6 +174,32 @@ public class ApplicationController {
     }
 
     /**
+     * Triggers reindexing for the given document types in the given clusters, for the given application.
+     *
+     * If no clusters are given, reindexing is triggered for the entire application; otherwise
+     * if no documents types are given, reindexing is triggered for all given clusters; otherwise
+     * reindexing is triggered for the cartesian product of the given clusters and document types.
+     */
+    public void reindex(ApplicationId id, ZoneId zoneId, List<String> clusterNames, List<String> documentTypes) {
+        configServer.reindex(new DeploymentId(id, zoneId), clusterNames, documentTypes);
+    }
+
+    /** Returns the reindexing status for the given application in the given zone. */
+    public ApplicationReindexing applicationReindexing(ApplicationId id, ZoneId zoneId) {
+        return configServer.getReindexing(new DeploymentId(id, zoneId));
+    }
+
+    /** Enables reindexing for the given application in the given zone. */
+    public void enableReindexing(ApplicationId id, ZoneId zoneId) {
+        configServer.enableReindexing(new DeploymentId(id, zoneId));
+    }
+
+    /** Disables reindexing for the given application in the given zone. */
+    public void disableReindexing(ApplicationId id, ZoneId zoneId) {
+        configServer.disableReindexing(new DeploymentId(id, zoneId));
+    }
+
+    /**
      * Returns the application with the given id
      *
      * @throws IllegalArgumentException if it does not exist
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 7b1b0f2f36e..0f9d3e181b4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
@@ -129,12 +130,15 @@ import java.util.Scanner;
 import java.util.StringJoiner;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
 import static com.yahoo.jdisc.Response.Status.CONFLICT;
 import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
 import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
+import static java.util.Map.Entry.comparingByKey;
 import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toUnmodifiableList;
 
 /**
  * This implements the application/v4 API which is used to deploy and manage applications
@@ -234,6 +238,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return getReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request);
@@ -284,6 +289,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindex")) return reindex(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return enableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above
@@ -311,6 +318,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return resume(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/reindexing")) return disableReindexing(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request);
@@ -1531,6 +1539,83 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new MessageResponse(response.toString());
     }
 
+    /** Schedule reindexing of an application, or a subset of clusters, possibly on a subset of documents. */
+    private HttpResponse reindex(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+        ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
+        ZoneId zone = ZoneId.from(environment, region);
+        List<String> clusterNames = Optional.ofNullable(request.getProperty("cluster")).stream()
+                                            .flatMap(clusters -> Stream.of(clusters.split(",")))
+                                            .filter(cluster -> ! cluster.isBlank())
+                                            .collect(toUnmodifiableList());
+        List<String> documentTypes = Optional.ofNullable(request.getProperty("type")).stream()
+                                             .flatMap(types -> Stream.of(types.split(",")))
+                                             .filter(type -> ! type.isBlank())
+                                             .collect(toUnmodifiableList());
+
+        controller.applications().reindex(id, zone, clusterNames, documentTypes);
+        return new MessageResponse("Requested reindexing of " + id + " in " + zone +
+                                   (clusterNames.isEmpty() ? "" : ", on clusters " + String.join(", ", clusterNames) +
+                                                                  (documentTypes.isEmpty() ? "" : ", for types " + String.join(", ", documentTypes))));
+    }
+
+    /** Gets reindexing status of an application in a zone. */
+    private HttpResponse getReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+        ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
+        ZoneId zone = ZoneId.from(environment, region);
+        ApplicationReindexing reindexing = controller.applications().applicationReindexing(id, zone);
+
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+
+        root.setBool("enabled", reindexing.enabled());
+        setStatus(root.setObject("status"), reindexing.common());
+
+        Cursor clustersArray = root.setArray("clusters");
+        reindexing.clusters().entrySet().stream().sorted(comparingByKey())
+                  .forEach(cluster -> {
+                      Cursor clusterObject = clustersArray.addObject();
+                      clusterObject.setString("name", cluster.getKey());
+                      setStatus(clusterObject.setObject("status"), cluster.getValue().common());
+
+                      Cursor pendingArray = clusterObject.setArray("pending");
+                      cluster.getValue().pending().entrySet().stream().sorted(comparingByKey())
+                             .forEach(pending -> {
+                                 Cursor pendingObject = pendingArray.addObject();
+                                 pendingObject.setString("type", pending.getKey());
+                                 pendingObject.setLong("requiredGeneration", pending.getValue());
+                             });
+
+                      Cursor readyArray = clusterObject.setArray("ready");
+                      cluster.getValue().ready().entrySet().stream().sorted(comparingByKey())
+                             .forEach(ready -> {
+                                 Cursor readyObject = readyArray.addObject();
+                                 readyObject.setString("type", ready.getKey());
+                                 setStatus(readyObject, ready.getValue());
+                             });
+                  });
+        return new SlimeJsonResponse(slime);
+    }
+
+    void setStatus(Cursor statusObject, ApplicationReindexing.Status status) {
+        statusObject.setLong("readyAtMillis", status.readyAt().toEpochMilli());
+    }
+
+    /** Enables reindexing of an application in a zone. */
+    private HttpResponse enableReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+        ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
+        ZoneId zone = ZoneId.from(environment, region);
+        controller.applications().enableReindexing(id, zone);
+        return new MessageResponse("Enabled reindexing of " + id + " in " + zone);
+    }
+
+    /** Disables reindexing of an application in a zone. */
+    private HttpResponse disableReindexing(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
+        ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName);
+        ZoneId zone = ZoneId.from(environment, region);
+        controller.applications().disableReindexing(id, zone);
+        return new MessageResponse("Disabled reindexing of " + id + " in " + zone);
+    }
+
     /** Schedule restart of deployment, or specific host in a deployment */
     private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
         DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 1465e1775dd..f48b6f1f687 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -22,6 +22,8 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing.Status;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
@@ -414,6 +416,25 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
+    public void reindex(DeploymentId deployment, List<String> clusterNames, List<String> documentTypes) { }
+
+    @Override
+    public ApplicationReindexing getReindexing(DeploymentId deployment) {
+        return new ApplicationReindexing(true,
+                                         new Status(Instant.ofEpochMilli(123)),
+                                         Map.of("cluster",
+                                                new ApplicationReindexing.Cluster(new Status(Instant.ofEpochMilli(234)),
+                                                                                  Map.of("type", 100L),
+                                                                                  Map.of("type", new Status(Instant.ofEpochMilli(345))))));
+    }
+
+    @Override
+    public void disableReindexing(DeploymentId deployment) { }
+
+    @Override
+    public void enableReindexing(DeploymentId deployment) { }
+
+    @Override
     public boolean isSuspended(DeploymentId deployment) {
         return suspendedApplications.contains(deployment);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 8e74542ec23..e5d50c7d02d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -580,6 +580,39 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .userIdentity(USER_ID),
                               "{\"message\":\"Triggered production-us-west-1 for tenant1.application1.instance1\"}");
 
+        // POST a 'reindex application' command
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindex", POST)
+                                      .userIdentity(USER_ID),
+                              "{\"message\":\"Requested reindexing of tenant1.application1.instance1 in prod.us-central-1\"}");
+
+        // POST a 'reindex application' command with cluster filter
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindex", POST)
+                                      .properties(Map.of("cluster", "boo,moo"))
+                                      .userIdentity(USER_ID),
+                              "{\"message\":\"Requested reindexing of tenant1.application1.instance1 in prod.us-central-1, on clusters boo, moo\"}");
+
+        // POST a 'reindex application' command with cluster and document type filters
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindex", POST)
+                                      .properties(Map.of("cluster", "boo,moo",
+                                                         "type", "foo,boo"))
+                                      .userIdentity(USER_ID),
+                              "{\"message\":\"Requested reindexing of tenant1.application1.instance1 in prod.us-central-1, on clusters boo, moo, for types foo, boo\"}");
+
+        // POST to enable reindexing
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindexing", POST)
+                                      .userIdentity(USER_ID),
+                              "{\"message\":\"Enabled reindexing of tenant1.application1.instance1 in prod.us-central-1\"}");
+
+        // DELETE to disable reindexing
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindexing", DELETE)
+                                      .userIdentity(USER_ID),
+                              "{\"message\":\"Disabled reindexing of tenant1.application1.instance1 in prod.us-central-1\"}");
+
+        // GET to get reindexing status
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindexing", GET)
+                                      .userIdentity(USER_ID),
+                              "{\"enabled\":true,\"status\":{\"readyAtMillis\":123},\"clusters\":[{\"name\":\"cluster\",\"status\":{\"readyAtMillis\":234},\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345}]}]}");
+
         // POST a 'restart application' command
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST)
                                       .userIdentity(USER_ID),