Merge pull request #15290 from vespa-engine/jonmv/reindexing-status-in-application-v2

Jonmv/reindexing status in application v2

7 files changed, 362 insertions, 74 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 7a0dd1cf92a..b22e386aac4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -1054,6 +1054,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                         RegionName.from(configserverConfig.region()));
     }
 
+    public Clock clock() { return clock; }
+
     /** Emits as a metric the time in millis spent while holding this timer, with deployment ID as dimensions. */
     public ActionTimer timerFor(ApplicationId id, String metricName) {
         return new ActionTimer(metric, clock, id, configserverConfig.environment(), configserverConfig.region(), metricName);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
index db9b81d0657..cb81bed155e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
@@ -4,12 +4,14 @@ package com.yahoo.vespa.config.server.application;
 import com.yahoo.config.model.api.Reindexing;
 
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Stream;
 
 import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableMap;
 
 /**
@@ -37,15 +39,22 @@ public class ApplicationReindexing implements Reindexing {
         return new ApplicationReindexing(true, new Status(now), Map.of());
     }
 
-    /** Returns a copy of this with common reindexing for the whole application ready at the given instant. */
+    /** Returns a copy of this with reindexing for the whole application ready at the given instant. */
     public ApplicationReindexing withReady(Instant readyAt) {
-        return new ApplicationReindexing(enabled, new Status(readyAt), clusters);
+        return new ApplicationReindexing(enabled,
+                                         new Status(readyAt),
+                                         clusters.entrySet().stream()
+                                                 .filter(cluster -> ! cluster.getValue().pending.isEmpty())
+                                                 .collect(toUnmodifiableMap(cluster -> cluster.getKey(),
+                                                                            cluster -> new Cluster(new Status(readyAt),
+                                                                                                   cluster.getValue().pending,
+                                                                                                   Map.of()))));
     }
 
-    /** Returns a copy of this with common reindexing for the given cluster ready at the given instant. */
+    /** Returns a copy of this with reindexing for the given cluster ready at the given instant. */
     public ApplicationReindexing withReady(String cluster, Instant readyAt) {
         Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common));
-        Cluster modified = new Cluster(new Status(readyAt), current.pending, current.ready);
+        Cluster modified = new Cluster(new Status(readyAt), current.pending, Map.of());
         return new ApplicationReindexing(enabled, common, with(cluster, modified, clusters));
     }
 
@@ -53,7 +62,7 @@ public class ApplicationReindexing implements Reindexing {
     public ApplicationReindexing withReady(String cluster, String documentType, Instant readyAt) {
         Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common));
         Cluster modified = new Cluster(current.common,
-                                       without(documentType, current.pending),
+                                       current.pending,
                                        with(documentType, new Status(readyAt), current.ready));
         return new ApplicationReindexing(enabled, common, with(cluster, modified, clusters));
     }
@@ -63,7 +72,16 @@ public class ApplicationReindexing implements Reindexing {
         Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common));
         Cluster modified = new Cluster(current.common,
                                        with(documentType, requirePositive(requiredGeneration), current.pending),
-                                       without(documentType, current.ready));
+                                       current.ready);
+        return new ApplicationReindexing(enabled, common, with(cluster, modified, clusters));
+    }
+
+    /** Returns a copy of this with no pending reindexing for the given document type. */
+    public ApplicationReindexing withoutPending(String cluster, String documentType) {
+        Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common));
+        Cluster modified = new Cluster(current.common,
+                                       without(documentType, current.pending),
+                                       current.ready);
         return new ApplicationReindexing(enabled, common, with(cluster, modified, clusters));
     }
 
@@ -87,16 +105,11 @@ public class ApplicationReindexing implements Reindexing {
 
     @Override
     public Optional<Reindexing.Status> status(String cluster, String documentType) {
-        if (clusters.containsKey(cluster)) {
-            if (clusters.get(cluster).pending().containsKey(documentType))
-                return Optional.empty();
-
-            if (clusters.get(cluster).ready().containsKey(documentType))
-                return Optional.of(clusters.get(cluster).ready().get(documentType));
-
-            return Optional.of(clusters.get(cluster).common());
-        }
-        return Optional.of(common());
+        return ! clusters.containsKey(cluster)
+               ? Optional.of(common())
+               : ! clusters.get(cluster).ready().containsKey(documentType)
+                 ? Optional.of(clusters.get(cluster).common())
+                 : Optional.of(clusters.get(cluster).ready().get(documentType));
     }
 
     @Override
@@ -186,7 +199,7 @@ public class ApplicationReindexing implements Reindexing {
         private final Instant ready;
 
         Status(Instant ready) {
-            this.ready = requireNonNull(ready);
+            this.ready = ready.truncatedTo(ChronoUnit.MILLIS);
         }
 
         @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 01d0938a9b0..f336f183d56 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -17,21 +17,28 @@ import com.yahoo.jdisc.application.BindingMatch;
 import com.yahoo.jdisc.application.UriPattern;
 import com.yahoo.slime.Cursor;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase;
+import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.http.ContentHandler;
 import com.yahoo.vespa.config.server.http.ContentRequest;
 import com.yahoo.vespa.config.server.http.HttpErrorResponse;
 import com.yahoo.vespa.config.server.http.HttpHandler;
 import com.yahoo.vespa.config.server.http.JSONResponse;
 import com.yahoo.vespa.config.server.http.NotFoundException;
+import com.yahoo.vespa.curator.Lock;
 
 import java.io.IOException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.util.Map.Entry.comparingByKey;
+import static java.util.stream.Collectors.toList;
+
 /**
  * Operations on applications (delete, wait for config convergence, restart, application content etc.)
  *
@@ -43,6 +50,8 @@ public class ApplicationHandler extends HttpHandler {
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart",
+            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindex",
+            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindexing",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*",
@@ -56,7 +65,7 @@ public class ApplicationHandler extends HttpHandler {
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*",
             "http://*/application/v2/tenant/*/application/*")
             .map(UriPattern::new)
-            .collect(Collectors.toList());
+            .collect(toList());
 
     private final Zone zone;
     private final ApplicationRepository applicationRepository;
@@ -73,10 +82,16 @@ public class ApplicationHandler extends HttpHandler {
     @Override
     public HttpResponse handleDELETE(HttpRequest request) {
         ApplicationId applicationId = getApplicationIdFromRequest(request);
-        boolean deleted = applicationRepository.delete(applicationId);
-        if ( ! deleted)
-            return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found");
-        return new DeleteApplicationResponse(Response.Status.OK, applicationId);
+
+        if (isReindexingRequest(request)) {
+            setReindexingEnabled(applicationId, false);
+            return new JSONResponse(Response.Status.OK);
+        }
+
+        if (applicationRepository.delete(applicationId))
+            return new DeleteApplicationResponse(Response.Status.OK, applicationId);
+
+        return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found");
     }
 
     @Override
@@ -97,6 +112,10 @@ public class ApplicationHandler extends HttpHandler {
             return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix);
         }
 
+        if (isReindexingRequest(request)) {
+            return getReindexingStatus(applicationId);
+        }
+
         if (isContentRequest(request)) {
             long sessionId = applicationRepository.getSessionIdForApplication(applicationId);
             String contentPath = getBindingMatch(request).group(7);
@@ -176,9 +195,11 @@ public class ApplicationHandler extends HttpHandler {
     @Override
     public HttpResponse handlePOST(HttpRequest request) {
         ApplicationId applicationId = getApplicationIdFromRequest(request);
-        if (request.getUri().getPath().endsWith("restart")) {
+
+        if (isRestartRequest(request))
             return restart(request, applicationId);
-        } else if (isTesterStartTestsRequest(request)) {
+
+        if (isTesterStartTestsRequest(request)) {
             byte[] data;
             try {
                 data = IOUtils.readBytes(request.getData(), 1024 * 1000);
@@ -186,15 +207,68 @@ public class ApplicationHandler extends HttpHandler {
                 throw new IllegalArgumentException("Could not read data in request " + request);
             }
             return applicationRepository.startTests(applicationId, getSuiteFromRequest(request), data);
-        } else {
-            throw new NotFoundException("Illegal POST request '" + request.getUri() + "'");
+        }
+
+        if (isReindexRequest(request)) {
+            triggerReindexing(request, applicationId);
+            return new JSONResponse(Response.Status.OK);
+        }
+
+        if (isReindexingRequest(request)) {
+            setReindexingEnabled(applicationId, true);
+            return new JSONResponse(Response.Status.OK);
+        }
+
+        throw new NotFoundException("Illegal POST request '" + request.getUri() + "'");
+    }
+
+    private void triggerReindexing(HttpRequest request, ApplicationId applicationId) {
+        List<String> clusters = Optional.ofNullable(request.getProperty("cluster")).stream()
+                                        .flatMap(value -> Stream.of(value.split(",")))
+                                        .filter(cluster -> ! cluster.isBlank())
+                                        .collect(toList());
+        List<String> types = Optional.ofNullable(request.getProperty("type")).stream()
+                                     .flatMap(value -> Stream.of(value.split(",")))
+                                     .filter(type -> ! type.isBlank())
+                                     .collect(toList());
+        Instant now = applicationRepository.clock().instant();
+        ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
+        try (Lock lock = database.lock(applicationId)) {
+            ApplicationReindexing reindexing = database.readReindexingStatus(applicationId)
+                                                       .orElse(ApplicationReindexing.ready(now));
+            if (clusters.isEmpty())
+                reindexing = reindexing.withReady(now);
+            else
+                for (String cluster : clusters)
+                    if (types.isEmpty())
+                        reindexing = reindexing.withReady(cluster, now);
+                    else
+                        for (String type : types)
+                            reindexing = reindexing.withReady(cluster, type, now);
+            database.writeReindexingStatus(applicationId, reindexing);
+        }
+    }
+
+    void setReindexingEnabled(ApplicationId applicationId, boolean enabled) {
+        Instant now = applicationRepository.clock().instant();
+        ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
+        try (Lock lock = database.lock(applicationId)) {
+            database.writeReindexingStatus(applicationId,
+                                           database.readReindexingStatus(applicationId)
+                                                   .orElse(ApplicationReindexing.ready(now))
+                                                   .enabled(enabled));
         }
     }
 
+    private HttpResponse getReindexingStatus(ApplicationId applicationId) {
+        return new ReindexResponse(applicationRepository.getTenant(applicationId).getApplicationRepo().database()
+                                                        .readReindexingStatus(applicationId));
+    }
+
     private HttpResponse restart(HttpRequest request, ApplicationId applicationId) {
         if (getBindingMatch(request).groupCount() != 7)
             throw new NotFoundException("Illegal POST restart request '" + request.getUri() +
-                                        "': Must have 6 arguments but had " + ( getBindingMatch(request).groupCount()-1 ) );
+                                        "': Must have 6 arguments but had " + (getBindingMatch(request).groupCount() - 1));
         applicationRepository.restart(applicationId, hostFilterFrom(request));
         return new JSONResponse(Response.Status.OK); // return empty
     }
@@ -218,6 +292,21 @@ public class ApplicationHandler extends HttpHandler {
                 .orElseThrow(() -> new IllegalArgumentException("Illegal url for config request: " + request.getUri()));
     }
 
+    private static boolean isRestartRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+               request.getUri().getPath().endsWith("/restart");
+    }
+
+    private static boolean isReindexRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+               request.getUri().getPath().endsWith("/reindex");
+    }
+
+    private static boolean isReindexingRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+               request.getUri().getPath().endsWith("/reindexing");
+    }
+
     private static boolean isIsSuspendedRequest(HttpRequest request) {
         return getBindingMatch(request).groupCount() == 7 &&
                request.getUri().getPath().endsWith("/suspended");
@@ -360,4 +449,43 @@ public class ApplicationHandler extends HttpHandler {
             object.setDouble("rate", usageRate);
         }
     }
+
+    private static class ReindexResponse extends JSONResponse {
+        ReindexResponse(Optional<ApplicationReindexing> applicationReindexing) {
+            super(Response.Status.OK);
+            applicationReindexing.ifPresent(reindexing -> {
+                object.setBool("enabled", reindexing.enabled());
+                setStatus(object.setObject("status"), reindexing.common());
+
+                Cursor clustersArray = object.setArray("clusters");
+                reindexing.clusters().entrySet().stream().sorted(comparingByKey())
+                          .forEach(cluster -> {
+                              Cursor clusterObject = clustersArray.addObject();
+                              clusterObject.setString("name", cluster.getKey());
+                              setStatus(clusterObject.setObject("status"), cluster.getValue().common());
+
+                              Cursor pendingArray = clusterObject.setArray("pending");
+                              cluster.getValue().pending().entrySet().stream().sorted(comparingByKey())
+                                     .forEach(pending -> {
+                                         Cursor pendingObject = pendingArray.addObject();
+                                         pendingObject.setString("type", pending.getKey());
+                                         pendingObject.setLong("requiredGeneration", pending.getValue());
+                                     });
+
+                              Cursor readyArray = clusterObject.setArray("ready");
+                              cluster.getValue().ready().entrySet().stream().sorted(comparingByKey())
+                                     .forEach(ready -> {
+                                         Cursor readyObject = readyArray.addObject();
+                                         readyObject.setString("type", ready.getKey());
+                                         setStatus(readyObject, ready.getValue());
+                                     });
+                          });
+            });
+        }
+
+        private static void setStatus(Cursor object, ApplicationReindexing.Status status) {
+            object.setLong("readyMillis", status.ready().toEpochMilli());
+        }
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
index cc140f14bc5..4aa89bb8c93 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
@@ -16,12 +16,10 @@ import com.yahoo.yolean.Exceptions;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Collection;
 import java.util.Comparator;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -63,7 +61,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
                                              try (Lock lock = database.lock(id)) {
                                                  ApplicationReindexing reindexing = database.readReindexingStatus(id)
                                                                                             .orElse(ApplicationReindexing.ready(clock.instant()));
-                                                 database.writeReindexingStatus(id, withReady(reindexing, lazyGeneration(application), clock.instant()));
+                                                 database.writeReindexingStatus(id, withConvergenceOn(reindexing, lazyGeneration(application), clock.instant()));
                                              }
                                          }
                                          catch (RuntimeException e) {
@@ -87,11 +85,12 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
         };
     }
 
-    static ApplicationReindexing withReady(ApplicationReindexing reindexing, Supplier<Long> oldestGeneration, Instant now) {
+    static ApplicationReindexing withConvergenceOn(ApplicationReindexing reindexing, Supplier<Long> oldestGeneration, Instant now) {
         for (var cluster : reindexing.clusters().entrySet()) {
             for (var pending : cluster.getValue().pending().entrySet())
                 if (pending.getValue() <= oldestGeneration.get())
-                    reindexing = reindexing.withReady(cluster.getKey(), pending.getKey(), now);
+                    reindexing = reindexing.withReady(cluster.getKey(), pending.getKey(), now)
+                                           .withoutPending(cluster.getKey(), pending.getKey());
 
             for (var documentType : cluster.getValue().ready().entrySet())
                 if (documentType.getValue().ready().isBefore(now.minus(reindexingInterval)))
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java
index 3a72893aeb0..128ba9c0c9d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java
@@ -27,7 +27,8 @@ public class ApplicationReindexingTest {
                                                                 .withPending("two", "b", 20)
                                                                 .withReady("two", Instant.ofEpochMilli(2 << 10))
                                                                 .withReady("one", "a", Instant.ofEpochMilli(1))
-                                                                .withReady("two", "c", Instant.ofEpochMilli(3));
+                                                                .withReady("two", "c", Instant.ofEpochMilli(3))
+                                                                .withoutPending("one", "a");
 
         // Document is most specific, and is used.
         assertEquals(Instant.ofEpochMilli(1),
@@ -45,8 +46,9 @@ public class ApplicationReindexingTest {
         assertEquals(Instant.ofEpochMilli(1 << 20),
                      reindexing.status("three", "a").orElseThrow().ready());
 
-        assertEquals(Optional.empty(),
-                     reindexing.status("two", "b"));
+        // Cluster is most specific, and is used, also when pending reindexing exists for document.
+        assertEquals(Instant.ofEpochMilli(2 << 10),
+                     reindexing.status("two", "b").orElseThrow().ready());
 
         assertEquals(new Status(Instant.ofEpochMilli(1 << 20)),
                      reindexing.common());
@@ -75,7 +77,6 @@ public class ApplicationReindexingTest {
         assertTrue(reindexing.enabled());
 
         assertFalse(reindexing.enabled(false).enabled());
-
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 7276091fed0..e9bf4f7c12e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -13,11 +13,16 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.test.ManualClock;
+import com.yahoo.test.json.JsonTestHelper;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.MockLogRetriever;
 import com.yahoo.vespa.config.server.MockProvisioner;
 import com.yahoo.vespa.config.server.MockTesterClient;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
+import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase;
+import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.OrchestratorMock;
@@ -39,15 +44,22 @@ import org.junit.rules.TemporaryFolder;
 
 import javax.ws.rs.client.Client;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.List;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
+import static com.yahoo.container.jdisc.HttpRequest.createTestRequest;
+import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE;
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
+import static com.yahoo.jdisc.http.HttpRequest.Method.POST;
+import static com.yahoo.test.json.JsonTestHelper.assertJsonEquals;
+import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCodeAndMessage;
 import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -76,12 +88,14 @@ public class ApplicationHandlerTest {
     private ApplicationRepository applicationRepository;
     private MockProvisioner provisioner;
     private OrchestratorMock orchestrator;
+    private ManualClock clock;
 
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
     @Before
     public void setup() throws IOException {
+        clock = new ManualClock();
         List<ModelFactory> modelFactories = List.of(DeployTester.createModelFactory(vespaVersion));
         ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder()
                 .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath())
@@ -92,6 +106,7 @@ public class ApplicationHandlerTest {
                 .provisioner(provisioner)
                 .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories))
                 .configServerConfig(configserverConfig)
+                .clock(clock)
                 .build();
         tenantRepository = new TenantRepository(componentRegistry);
         tenantRepository.addTenant(mytenantName);
@@ -186,7 +201,7 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams).sessionId();
 
         var url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/quota";
-        var response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, GET));
+        var response = createApplicationHandler().handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         var renderedString = SessionHandlerTest.getRenderedString(response);
 
@@ -194,6 +209,110 @@ public class ApplicationHandlerTest {
     }
 
     @Test
+    public void testReindex() throws Exception {
+        ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
+        applicationRepository.deploy(testApp, prepareParams(applicationId));
+        ApplicationReindexing expected = ApplicationReindexing.ready(clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "");
+        expected = expected.withReady(clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        expected = expected.withReady(clock.instant());
+        reindex(applicationId, "?cluster=");
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        expected = expected.withReady(clock.instant());
+        reindex(applicationId, "?type=moo");
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "?cluster=foo,boo");
+        expected = expected.withReady("foo", clock.instant())
+                           .withReady("boo", clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "?cluster=foo,boo&type=bar,baz");
+        expected = expected.withReady("foo", "bar", clock.instant())
+                           .withReady("foo", "baz", clock.instant())
+                           .withReady("boo", "bar", clock.instant())
+                           .withReady("boo", "baz", clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        reindexing(applicationId, DELETE);
+        expected = expected.enabled(false);
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        reindexing(applicationId, POST);
+        expected = expected.enabled(true);
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        database.writeReindexingStatus(applicationId, expected.withPending("boo", "bar", 123L));
+
+        long now = clock.instant().toEpochMilli();
+        reindexing(applicationId, GET, "{" +
+                                       "  \"enabled\": true," +
+                                       "  \"status\": {" +
+                                       "    \"readyMillis\": " + (now - 2000) +
+                                       "  }," +
+                                       "  \"clusters\": [" +
+                                       "    {" +
+                                       "      \"name\": \"boo\"," +
+                                       "      \"status\": {" +
+                                       "        \"readyMillis\": " + (now - 1000) +
+                                       "      }," +
+                                       "      \"pending\": [" +
+                                       "        {" +
+                                       "          \"type\": \"bar\"," +
+                                       "          \"requiredGeneration\": 123" +
+                                       "        }" +
+                                       "      ]," +
+                                       "      \"ready\": [" +
+                                       "        {" +
+                                       "          \"type\": \"bar\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }," +
+                                       "        {" +
+                                       "          \"type\": \"baz\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }" +
+                                       "      ]" +
+                                       "    }," +
+                                       "    {" +
+                                       "      \"name\": \"foo\", " +
+                                       "      \"status\": {" +
+                                       "        \"readyMillis\": " + (now - 1000) +
+                                       "      }," +
+                                       "      \"pending\": []," +
+                                       "      \"ready\": [" +
+                                       "        {" +
+                                       "          \"type\": \"bar\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }," +
+                                       "        {" +
+                                       "          \"type\": \"baz\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }" +
+                                       "      ]" +
+                                       "    }" +
+                                       "  ]" +
+                                       "}");
+    }
+
+    @Test
     public void testRestart() throws Exception {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         assertFalse(provisioner.restarted());
@@ -233,13 +352,13 @@ public class ApplicationHandlerTest {
         when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1")))
                 .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));
 
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>");
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
+        assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>");
     }
 
     @Test
     public void testPutIsIllegal() throws IOException {
-        assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.PUT);
+        assertNotAllowed(Method.PUT);
     }
 
     @Test
@@ -266,7 +385,7 @@ public class ApplicationHandlerTest {
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/logs?from=100&to=200";
         ApplicationHandler mockHandler = createApplicationHandler();
 
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
 
         assertEquals("log line", getRenderedString(response));
@@ -277,7 +396,7 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status";
         ApplicationHandler mockHandler = createApplicationHandler();
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         assertEquals("OK", getRenderedString(response));
     }
@@ -288,7 +407,7 @@ public class ApplicationHandlerTest {
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/log?after=1234";
         ApplicationHandler mockHandler = createApplicationHandler();
 
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         assertEquals("log", getRenderedString(response));
     }
@@ -300,7 +419,7 @@ public class ApplicationHandlerTest {
         ApplicationHandler mockHandler = createApplicationHandler();
 
         InputStream requestData =  new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8));
-        HttpRequest testRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.POST, requestData);
+        HttpRequest testRequest = createTestRequest(url, POST, requestData);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());
     }
@@ -310,7 +429,7 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/ready";
         ApplicationHandler mockHandler = createApplicationHandler();
-        HttpRequest testRequest = HttpRequest.createTestRequest(url, GET);
+        HttpRequest testRequest = createTestRequest(url, GET);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());
     }
@@ -320,13 +439,13 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/report";
         ApplicationHandler mockHandler = createApplicationHandler();
-        HttpRequest testRequest = HttpRequest.createTestRequest(url, GET);
+        HttpRequest testRequest = createTestRequest(url, GET);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());
         assertEquals("report", getRenderedString(response));
     }
 
-    private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException {
+    private void assertNotAllowed(Method method) throws IOException {
         String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default";
         deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}",
                                 method);
@@ -347,18 +466,18 @@ public class ApplicationHandlerTest {
 
     private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException {
         String expectedResponse = "{\"message\":\"Application '" + applicationId + "' deleted\"}";
-        deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE);
+        deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, Method.DELETE);
     }
 
     private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse) throws IOException {
-        deleteAndAssertResponse(toUrlPath(applicationId, zone, true), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE);
+        deleteAndAssertResponse(toUrlPath(applicationId, zone, true), expectedStatus, errorCode, expectedResponse, Method.DELETE);
     }
 
-    private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException {
+    private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, Method method) throws IOException {
         ApplicationHandler handler = createApplicationHandler();
-        HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method));
+        HttpResponse response = handler.handle(createTestRequest(url, method));
         if (expectedStatus == 200) {
-            HandlerTest.assertHttpStatusCodeAndMessage(response, 200, expectedResponse);
+            assertHttpStatusCodeAndMessage(response, 200, expectedResponse);
         } else {
             HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, expectedStatus, errorCode, expectedResponse);
         }
@@ -371,8 +490,8 @@ public class ApplicationHandlerTest {
 
     private void assertSuspended(boolean expectedValue, ApplicationId application, Zone zone) throws IOException {
         String restartUrl = toUrlPath(application, zone, true) + "/suspended";
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, GET));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"suspended\":" + expectedValue + "}");
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(restartUrl, GET));
+        assertHttpStatusCodeAndMessage(response, 200, "{\"suspended\":" + expectedValue + "}");
     }
 
     private String toUrlPath(ApplicationId application, Zone zone, boolean fullAppIdInUrl) {
@@ -383,7 +502,7 @@ public class ApplicationHandlerTest {
     }
 
     private void assertApplicationResponse(String url, long expectedGeneration, Version expectedVersion) throws IOException {
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         String renderedString = SessionHandlerTest.getRenderedString(response);
         assertEquals("{\"generation\":" + expectedGeneration +
@@ -408,21 +527,41 @@ public class ApplicationHandlerTest {
                                                    GET);
     }
 
+    private void reindexing(ApplicationId application, Method method, String expectedBody) throws IOException {
+        String reindexingUrl = toUrlPath(application, Zone.defaultZone(), true) + "/reindexing";
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(reindexingUrl, method));
+        assertEquals(200, response.getStatus());
+        if (expectedBody != null) {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            response.render(out);
+            assertJsonEquals(out.toString(), expectedBody);
+        }
+    }
+
+    private void reindexing(ApplicationId application, Method method) throws IOException {
+        reindexing(application, method, null);
+    }
+
+    private void reindex(ApplicationId application, String query) throws IOException {
+        String reindexUrl = toUrlPath(application, Zone.defaultZone(), true) + "/reindex" + query;
+        assertHttpStatusCodeAndMessage(createApplicationHandler().handle(createTestRequest(reindexUrl, POST)), 200, "");
+    }
+
     private void restart(ApplicationId application, Zone zone) throws IOException {
         String restartUrl = toUrlPath(application, zone, true) + "/restart";
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.POST));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "");
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(restartUrl, POST));
+        assertHttpStatusCodeAndMessage(response, 200, "");
     }
 
     private void converge(ApplicationId application, Zone zone) throws IOException {
         String convergeUrl = toUrlPath(application, zone, true) + "/serviceconverge";
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(convergeUrl, GET));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "");
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(convergeUrl, GET));
+        assertHttpStatusCodeAndMessage(response, 200, "");
     }
 
     private HttpResponse fileDistributionStatus(ApplicationId application, Zone zone) {
         String restartUrl = toUrlPath(application, zone, true) + "/filedistributionstatus";
-        return createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, GET));
+        return createApplicationHandler().handle(createTestRequest(restartUrl, GET));
     }
 
     private static class MockStateApiFactory implements ConfigConvergenceChecker.StateApiFactory {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java
index 57582040552..27ca4db3d40 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java
@@ -6,7 +6,7 @@ import org.junit.Test;
 
 import java.time.Instant;
 
-import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.withReady;
+import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.withConvergenceOn;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -16,29 +16,35 @@ public class ReindexingMaintainerTest {
 
     @Test
     public void testReadyComputation() {
-        ApplicationReindexing reindexing = ApplicationReindexing.ready(Instant.EPOCH)
+        ApplicationReindexing reindexing = ApplicationReindexing.ready(Instant.ofEpochMilli(1 << 20))
                                                                 .withPending("one", "a", 10)
-                                                                .withReady("two", "b", Instant.ofEpochMilli(2))
                                                                 .withPending("two", "b", 20)
-                                                                .withReady("two", Instant.ofEpochMilli(2 << 10))
+                                                                .withReady("one", Instant.EPOCH)
                                                                 .withReady("one", "a", Instant.ofEpochMilli(1))
-                                                                .withReady("two", "c", Instant.ofEpochMilli(3))
-                                                                .withReady(Instant.ofEpochMilli(1 << 20));
+                                                                .withReady("two", Instant.ofEpochMilli(2 << 10))
+                                                                .withReady("two", "b", Instant.ofEpochMilli(2))
+                                                                .withReady("two", "c", Instant.ofEpochMilli(3));
 
+        // Nothing happens without convergence.
         assertEquals(reindexing,
-                     withReady(reindexing, () -> -1L, Instant.EPOCH));
+                     withConvergenceOn(reindexing, () -> -1L, Instant.EPOCH));
 
-        assertEquals(reindexing,
-                     withReady(reindexing, () -> 19L, Instant.EPOCH));
+        // Status for (one, a) changes, but not (two, b).
+
+        assertEquals(reindexing.withReady("one", "a", Instant.EPOCH).withoutPending("one", "a"),
+                     withConvergenceOn(reindexing, () -> 19L, Instant.EPOCH));
 
         Instant later = Instant.ofEpochMilli(2).plus(ReindexingMaintainer.reindexingInterval);
-        assertEquals(reindexing.withReady("one", later)         // Had EPOCH as previous, so is updated.
+        assertEquals(reindexing.withReady("one", later)         // Had EPOCH as previous, so is updated, overwriting status for "a".
                                .withReady("two", "b", later)    // Got config convergence, so is updated.
-                               .withReady("one", "a", later),   // Had EPOCH + 1 as previous, so is updated.
-                     withReady(reindexing, () -> 20L, later));
+                               .withoutPending("one", "a")
+                               .withoutPending("two", "b"),
+                     withConvergenceOn(reindexing, () -> 20L, later));
 
         // Verify generation supplier isn't called when no pending document types.
-        withReady(reindexing.withReady("two", "b", later), () -> { throw new AssertionError("not supposed to run"); }, later);
+        withConvergenceOn(reindexing.withoutPending("one", "a").withoutPending("two", "b"),
+                          () -> { throw new AssertionError("not supposed to run"); },
+                          later);
     }
 
 }