REST API for reindexing status on config server

4 files changed, 300 insertions, 36 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 7a0dd1cf92a..b22e386aac4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -1054,6 +1054,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                         RegionName.from(configserverConfig.region()));
     }
 
+    public Clock clock() { return clock; }
+
     /** Emits as a metric the time in millis spent while holding this timer, with deployment ID as dimensions. */
     public ActionTimer timerFor(ApplicationId id, String metricName) {
         return new ActionTimer(metric, clock, id, configserverConfig.environment(), configserverConfig.region(), metricName);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
index d860f7dbab6..cb81bed155e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.application;
 import com.yahoo.config.model.api.Reindexing;
 
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -198,7 +199,7 @@ public class ApplicationReindexing implements Reindexing {
         private final Instant ready;
 
         Status(Instant ready) {
-            this.ready = requireNonNull(ready);
+            this.ready = ready.truncatedTo(ChronoUnit.MILLIS);
         }
 
         @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 01d0938a9b0..3bfd44948cc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -17,21 +17,28 @@ import com.yahoo.jdisc.application.BindingMatch;
 import com.yahoo.jdisc.application.UriPattern;
 import com.yahoo.slime.Cursor;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase;
+import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.http.ContentHandler;
 import com.yahoo.vespa.config.server.http.ContentRequest;
 import com.yahoo.vespa.config.server.http.HttpErrorResponse;
 import com.yahoo.vespa.config.server.http.HttpHandler;
 import com.yahoo.vespa.config.server.http.JSONResponse;
 import com.yahoo.vespa.config.server.http.NotFoundException;
+import com.yahoo.vespa.curator.Lock;
 
 import java.io.IOException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.util.Map.Entry.comparingByKey;
+import static java.util.stream.Collectors.toList;
+
 /**
  * Operations on applications (delete, wait for config convergence, restart, application content etc.)
  *
@@ -43,6 +50,8 @@ public class ApplicationHandler extends HttpHandler {
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/content/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/filedistributionstatus",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/restart",
+            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindex",
+            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/reindexing",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/suspended",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/serviceconverge/*",
@@ -56,7 +65,7 @@ public class ApplicationHandler extends HttpHandler {
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*",
             "http://*/application/v2/tenant/*/application/*")
             .map(UriPattern::new)
-            .collect(Collectors.toList());
+            .collect(toList());
 
     private final Zone zone;
     private final ApplicationRepository applicationRepository;
@@ -73,10 +82,16 @@ public class ApplicationHandler extends HttpHandler {
     @Override
     public HttpResponse handleDELETE(HttpRequest request) {
         ApplicationId applicationId = getApplicationIdFromRequest(request);
-        boolean deleted = applicationRepository.delete(applicationId);
-        if ( ! deleted)
-            return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found");
-        return new DeleteApplicationResponse(Response.Status.OK, applicationId);
+
+        if (isReindexingRequest(request)) {
+            setReindexingEnabled(applicationId, false);
+            return new JSONResponse(Response.Status.OK);
+        }
+
+        if (applicationRepository.delete(applicationId))
+            return new DeleteApplicationResponse(Response.Status.OK, applicationId);
+
+        return HttpErrorResponse.notFoundError("Unable to delete " + applicationId.toFullString() + ": Not found");
     }
 
     @Override
@@ -97,6 +112,10 @@ public class ApplicationHandler extends HttpHandler {
             return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix);
         }
 
+        if (isReindexingRequest(request)) {
+            return getReindexingStatus(applicationId);
+        }
+
         if (isContentRequest(request)) {
             long sessionId = applicationRepository.getSessionIdForApplication(applicationId);
             String contentPath = getBindingMatch(request).group(7);
@@ -176,9 +195,11 @@ public class ApplicationHandler extends HttpHandler {
     @Override
     public HttpResponse handlePOST(HttpRequest request) {
         ApplicationId applicationId = getApplicationIdFromRequest(request);
-        if (request.getUri().getPath().endsWith("restart")) {
+
+        if (isRestartRequest(request))
             return restart(request, applicationId);
-        } else if (isTesterStartTestsRequest(request)) {
+
+        if (isTesterStartTestsRequest(request)) {
             byte[] data;
             try {
                 data = IOUtils.readBytes(request.getData(), 1024 * 1000);
@@ -186,15 +207,68 @@ public class ApplicationHandler extends HttpHandler {
                 throw new IllegalArgumentException("Could not read data in request " + request);
             }
             return applicationRepository.startTests(applicationId, getSuiteFromRequest(request), data);
-        } else {
-            throw new NotFoundException("Illegal POST request '" + request.getUri() + "'");
+        }
+
+        if (isReindexRequest(request)) {
+            triggerReindexing(request, applicationId);
+            return new JSONResponse(Response.Status.OK);
+        }
+
+        if (isReindexingRequest(request)) {
+            setReindexingEnabled(applicationId, true);
+            return new JSONResponse(Response.Status.OK);
+        }
+
+        throw new NotFoundException("Illegal POST request '" + request.getUri() + "'");
+    }
+
+    private void triggerReindexing(HttpRequest request, ApplicationId applicationId) {
+        List<String> clusters = Optional.ofNullable(request.getProperty("cluster")).stream()
+                                        .flatMap(value -> Stream.of(value.split(",")))
+                                        .filter(cluster -> ! cluster.isBlank())
+                                        .collect(toList());
+        List<String> types = Optional.ofNullable(request.getProperty("type")).stream()
+                                     .flatMap(value -> Stream.of(value.split(",")))
+                                     .filter(type -> ! type.isBlank())
+                                     .collect(toList());
+        Instant now = applicationRepository.clock().instant();
+        ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
+        try (Lock lock = database.lock(applicationId)) {
+            ApplicationReindexing reindexing = database.readReindexingStatus(applicationId)
+                                                       .orElse(ApplicationReindexing.ready(now));
+            if (clusters.isEmpty())
+                reindexing = reindexing.withReady(now);
+            else
+                for (String cluster : clusters)
+                    if (types.isEmpty())
+                        reindexing = reindexing.withReady(cluster, now);
+                    else
+                        for (String type : types)
+                            reindexing = reindexing.withReady(cluster, type, now);
+            database.writeReindexingStatus(applicationId, reindexing);
+        }
+    }
+
+    void setReindexingEnabled(ApplicationId applicationId, boolean enabled) {
+        Instant now = applicationRepository.clock().instant();
+        ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
+        try (Lock lock = database.lock(applicationId)) {
+            database.writeReindexingStatus(applicationId,
+                                           database.readReindexingStatus(applicationId)
+                                                   .orElse(ApplicationReindexing.ready(now))
+                                                   .enabled(enabled));
         }
     }
 
+    private HttpResponse getReindexingStatus(ApplicationId applicationId) {
+        return new ReindexResponse(applicationRepository.getTenant(applicationId).getApplicationRepo().database()
+                                                        .readReindexingStatus(applicationId));
+    }
+
     private HttpResponse restart(HttpRequest request, ApplicationId applicationId) {
         if (getBindingMatch(request).groupCount() != 7)
             throw new NotFoundException("Illegal POST restart request '" + request.getUri() +
-                                        "': Must have 6 arguments but had " + ( getBindingMatch(request).groupCount()-1 ) );
+                                        "': Must have 6 arguments but had " + (getBindingMatch(request).groupCount() - 1));
         applicationRepository.restart(applicationId, hostFilterFrom(request));
         return new JSONResponse(Response.Status.OK); // return empty
     }
@@ -218,6 +292,21 @@ public class ApplicationHandler extends HttpHandler {
                 .orElseThrow(() -> new IllegalArgumentException("Illegal url for config request: " + request.getUri()));
     }
 
+    private static boolean isRestartRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+               request.getUri().getPath().endsWith("/restart");
+    }
+
+    private static boolean isReindexRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+               request.getUri().getPath().endsWith("/reindex");
+    }
+
+    private static boolean isReindexingRequest(HttpRequest request) {
+        return getBindingMatch(request).groupCount() == 7 &&
+               request.getUri().getPath().endsWith("/reindexing");
+    }
+
     private static boolean isIsSuspendedRequest(HttpRequest request) {
         return getBindingMatch(request).groupCount() == 7 &&
                request.getUri().getPath().endsWith("/suspended");
@@ -360,4 +449,43 @@ public class ApplicationHandler extends HttpHandler {
             object.setDouble("rate", usageRate);
         }
     }
+
+    private static class ReindexResponse extends JSONResponse {
+        ReindexResponse(Optional<ApplicationReindexing> applicationReindexing) {
+            super(Response.Status.OK);
+            applicationReindexing.ifPresent(reindexing -> {
+                object.setBool("enabled", reindexing.enabled());
+                setStatus(object.setObject("status"), reindexing.common());
+
+                Cursor clustersArray = object.setArray("clusters");
+                reindexing.clusters().entrySet().stream().sorted(comparingByKey())
+                          .forEach(cluster -> {
+                              Cursor clusterObject = clustersArray.addObject();
+                              clusterObject.setString("name", cluster.getKey());
+                              setStatus(clusterObject.setObject("status"), cluster.getValue().common());
+
+                              Cursor pendingArray = clusterObject.setArray("pending");
+                              cluster.getValue().pending().entrySet().stream().sorted(comparingByKey())
+                                     .forEach(pending -> {
+                                         Cursor pendingObject = pendingArray.addObject();
+                                         pendingObject.setString("type", pending.getKey());
+                                         pendingObject.setLong("requireGeneration", pending.getValue());
+                                     });
+
+                              Cursor readyArray = clusterObject.setArray("ready");
+                              cluster.getValue().ready().entrySet().stream().sorted(comparingByKey())
+                                     .forEach(ready -> {
+                                         Cursor readyObject = readyArray.addObject();
+                                         readyObject.setString("type", ready.getKey());
+                                         setStatus(readyObject, ready.getValue());
+                                     });
+                          });
+            });
+        }
+
+        private static void setStatus(Cursor object, ApplicationReindexing.Status status) {
+            object.setLong("readyMillis", status.ready().toEpochMilli());
+        }
+    }
+
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 7276091fed0..709ca35ad1c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -13,11 +13,16 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.http.HttpRequest.Method;
+import com.yahoo.test.ManualClock;
+import com.yahoo.test.json.JsonTestHelper;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.MockLogRetriever;
 import com.yahoo.vespa.config.server.MockProvisioner;
 import com.yahoo.vespa.config.server.MockTesterClient;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
+import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase;
+import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.application.HttpProxy;
 import com.yahoo.vespa.config.server.application.OrchestratorMock;
@@ -39,15 +44,22 @@ import org.junit.rules.TemporaryFolder;
 
 import javax.ws.rs.client.Client;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.List;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
+import static com.yahoo.container.jdisc.HttpRequest.createTestRequest;
+import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE;
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
+import static com.yahoo.jdisc.http.HttpRequest.Method.POST;
+import static com.yahoo.test.json.JsonTestHelper.assertJsonEquals;
+import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCodeAndMessage;
 import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -76,12 +88,14 @@ public class ApplicationHandlerTest {
     private ApplicationRepository applicationRepository;
     private MockProvisioner provisioner;
     private OrchestratorMock orchestrator;
+    private ManualClock clock;
 
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
     @Before
     public void setup() throws IOException {
+        clock = new ManualClock();
         List<ModelFactory> modelFactories = List.of(DeployTester.createModelFactory(vespaVersion));
         ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder()
                 .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath())
@@ -92,6 +106,7 @@ public class ApplicationHandlerTest {
                 .provisioner(provisioner)
                 .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories))
                 .configServerConfig(configserverConfig)
+                .clock(clock)
                 .build();
         tenantRepository = new TenantRepository(componentRegistry);
         tenantRepository.addTenant(mytenantName);
@@ -186,7 +201,7 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams).sessionId();
 
         var url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/quota";
-        var response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, GET));
+        var response = createApplicationHandler().handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         var renderedString = SessionHandlerTest.getRenderedString(response);
 
@@ -194,6 +209,103 @@ public class ApplicationHandlerTest {
     }
 
     @Test
+    public void testReindex() throws Exception {
+        ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
+        applicationRepository.deploy(testApp, prepareParams(applicationId));
+        ApplicationReindexing expected = ApplicationReindexing.ready(clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "");
+        expected = expected.withReady(clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        expected = expected.withReady(clock.instant());
+        reindex(applicationId, "?cluster=");
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        expected = expected.withReady(clock.instant());
+        reindex(applicationId, "?type=moo");
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "?cluster=foo,boo");
+        expected = expected.withReady("foo", clock.instant())
+                           .withReady("boo", clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "?cluster=foo,boo&type=bar,baz");
+        expected = expected.withReady("foo", "bar", clock.instant())
+                           .withReady("foo", "baz", clock.instant())
+                           .withReady("boo", "bar", clock.instant())
+                           .withReady("boo", "baz", clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        reindexing(applicationId, DELETE);
+        expected = expected.enabled(false);
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        reindexing(applicationId, POST);
+        expected = expected.enabled(true);
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        long now = clock.instant().toEpochMilli();
+        reindexing(applicationId, GET, "{" +
+                                       "  \"enabled\": true," +
+                                       "  \"status\": {" +
+                                       "    \"readyMillis\": " + (now - 2000) +
+                                       "  }," +
+                                       "  \"clusters\": [" +
+                                       "    {" +
+                                       "      \"name\": \"boo\"," +
+                                       "      \"status\": {" +
+                                       "        \"readyMillis\": " + (now - 1000) +
+                                       "      }," +
+                                       "      \"pending\": []," +
+                                       "      \"ready\": [" +
+                                       "        {" +
+                                       "          \"type\": \"bar\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }," +
+                                       "        {" +
+                                       "          \"type\": \"baz\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }" +
+                                       "      ]" +
+                                       "    }," +
+                                       "    {" +
+                                       "      \"name\": \"foo\", " +
+                                       "      \"status\": {" +
+                                       "        \"readyMillis\": " + (now - 1000) +
+                                       "      }," +
+                                       "      \"pending\": []," +
+                                       "      \"ready\": [" +
+                                       "        {" +
+                                       "          \"type\": \"bar\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }," +
+                                       "        {" +
+                                       "          \"type\": \"baz\"," +
+                                       "          \"readyMillis\": " + now +
+                                       "        }" +
+                                       "      ]" +
+                                       "    }" +
+                                       "  ]" +
+                                       "}");
+    }
+
+    @Test
     public void testRestart() throws Exception {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         assertFalse(provisioner.restarted());
@@ -233,13 +345,13 @@ public class ApplicationHandlerTest {
         when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1")))
                 .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));
 
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>");
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
+        assertHttpStatusCodeAndMessage(response, 200, "text/html", "<html>...</html>");
     }
 
     @Test
     public void testPutIsIllegal() throws IOException {
-        assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method.PUT);
+        assertNotAllowed(Method.PUT);
     }
 
     @Test
@@ -266,7 +378,7 @@ public class ApplicationHandlerTest {
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/logs?from=100&to=200";
         ApplicationHandler mockHandler = createApplicationHandler();
 
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
 
         assertEquals("log line", getRenderedString(response));
@@ -277,7 +389,7 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status";
         ApplicationHandler mockHandler = createApplicationHandler();
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         assertEquals("OK", getRenderedString(response));
     }
@@ -288,7 +400,7 @@ public class ApplicationHandlerTest {
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/log?after=1234";
         ApplicationHandler mockHandler = createApplicationHandler();
 
-        HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = mockHandler.handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         assertEquals("log", getRenderedString(response));
     }
@@ -300,7 +412,7 @@ public class ApplicationHandlerTest {
         ApplicationHandler mockHandler = createApplicationHandler();
 
         InputStream requestData =  new ByteArrayInputStream("foo".getBytes(StandardCharsets.UTF_8));
-        HttpRequest testRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.POST, requestData);
+        HttpRequest testRequest = createTestRequest(url, POST, requestData);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());
     }
@@ -310,7 +422,7 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/ready";
         ApplicationHandler mockHandler = createApplicationHandler();
-        HttpRequest testRequest = HttpRequest.createTestRequest(url, GET);
+        HttpRequest testRequest = createTestRequest(url, GET);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());
     }
@@ -320,13 +432,13 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/report";
         ApplicationHandler mockHandler = createApplicationHandler();
-        HttpRequest testRequest = HttpRequest.createTestRequest(url, GET);
+        HttpRequest testRequest = createTestRequest(url, GET);
         HttpResponse response = mockHandler.handle(testRequest);
         assertEquals(200, response.getStatus());
         assertEquals("report", getRenderedString(response));
     }
 
-    private void assertNotAllowed(com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException {
+    private void assertNotAllowed(Method method) throws IOException {
         String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default";
         deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.errorCodes.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}",
                                 method);
@@ -347,18 +459,18 @@ public class ApplicationHandlerTest {
 
     private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException {
         String expectedResponse = "{\"message\":\"Application '" + applicationId + "' deleted\"}";
-        deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE);
+        deleteAndAssertResponse(toUrlPath(applicationId, zone, fullAppIdInUrl), expectedStatus, errorCode, expectedResponse, Method.DELETE);
     }
 
     private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse) throws IOException {
-        deleteAndAssertResponse(toUrlPath(applicationId, zone, true), expectedStatus, errorCode, expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method.DELETE);
+        deleteAndAssertResponse(toUrlPath(applicationId, zone, true), expectedStatus, errorCode, expectedResponse, Method.DELETE);
     }
 
-    private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, com.yahoo.jdisc.http.HttpRequest.Method method) throws IOException {
+    private void deleteAndAssertResponse(String url, int expectedStatus, HttpErrorResponse.errorCodes errorCode, String expectedResponse, Method method) throws IOException {
         ApplicationHandler handler = createApplicationHandler();
-        HttpResponse response = handler.handle(HttpRequest.createTestRequest(url, method));
+        HttpResponse response = handler.handle(createTestRequest(url, method));
         if (expectedStatus == 200) {
-            HandlerTest.assertHttpStatusCodeAndMessage(response, 200, expectedResponse);
+            assertHttpStatusCodeAndMessage(response, 200, expectedResponse);
         } else {
             HandlerTest.assertHttpStatusCodeErrorCodeAndMessage(response, expectedStatus, errorCode, expectedResponse);
         }
@@ -371,8 +483,8 @@ public class ApplicationHandlerTest {
 
     private void assertSuspended(boolean expectedValue, ApplicationId application, Zone zone) throws IOException {
         String restartUrl = toUrlPath(application, zone, true) + "/suspended";
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, GET));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "{\"suspended\":" + expectedValue + "}");
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(restartUrl, GET));
+        assertHttpStatusCodeAndMessage(response, 200, "{\"suspended\":" + expectedValue + "}");
     }
 
     private String toUrlPath(ApplicationId application, Zone zone, boolean fullAppIdInUrl) {
@@ -383,7 +495,7 @@ public class ApplicationHandlerTest {
     }
 
     private void assertApplicationResponse(String url, long expectedGeneration, Version expectedVersion) throws IOException {
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(url, GET));
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(url, GET));
         assertEquals(200, response.getStatus());
         String renderedString = SessionHandlerTest.getRenderedString(response);
         assertEquals("{\"generation\":" + expectedGeneration +
@@ -408,21 +520,42 @@ public class ApplicationHandlerTest {
                                                    GET);
     }
 
+    private void reindexing(ApplicationId application, Method method, String expectedBody) throws IOException {
+        String reindexingUrl = toUrlPath(application, Zone.defaultZone(), true) + "/reindexing";
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(reindexingUrl, method));
+        assertEquals(200, response.getStatus());
+        if (expectedBody != null) {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            response.render(out);
+            System.err.println(out);
+            assertJsonEquals(out.toString(), expectedBody);
+        }
+    }
+
+    private void reindexing(ApplicationId application, Method method) throws IOException {
+        reindexing(application, method, null);
+    }
+
+    private void reindex(ApplicationId application, String query) throws IOException {
+        String reindexUrl = toUrlPath(application, Zone.defaultZone(), true) + "/reindex" + query;
+        assertHttpStatusCodeAndMessage(createApplicationHandler().handle(createTestRequest(reindexUrl, POST)), 200, "");
+    }
+
     private void restart(ApplicationId application, Zone zone) throws IOException {
         String restartUrl = toUrlPath(application, zone, true) + "/restart";
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, com.yahoo.jdisc.http.HttpRequest.Method.POST));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "");
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(restartUrl, POST));
+        assertHttpStatusCodeAndMessage(response, 200, "");
     }
 
     private void converge(ApplicationId application, Zone zone) throws IOException {
         String convergeUrl = toUrlPath(application, zone, true) + "/serviceconverge";
-        HttpResponse response = createApplicationHandler().handle(HttpRequest.createTestRequest(convergeUrl, GET));
-        HandlerTest.assertHttpStatusCodeAndMessage(response, 200, "");
+        HttpResponse response = createApplicationHandler().handle(createTestRequest(convergeUrl, GET));
+        assertHttpStatusCodeAndMessage(response, 200, "");
     }
 
     private HttpResponse fileDistributionStatus(ApplicationId application, Zone zone) {
         String restartUrl = toUrlPath(application, zone, true) + "/filedistributionstatus";
-        return createApplicationHandler().handle(HttpRequest.createTestRequest(restartUrl, GET));
+        return createApplicationHandler().handle(createTestRequest(restartUrl, GET));
     }
 
     private static class MockStateApiFactory implements ConfigConvergenceChecker.StateApiFactory {