Let all ApplicationReindexing users config-aware

This allows checking for valid clusters and document types when manipulating status.
This commit also updates responses in the application/v2/ API to reflect this

9 files changed, 190 insertions, 58 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
index 2828b4c62e0..1736b23012d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
@@ -2,16 +2,28 @@
 package com.yahoo.vespa.config.server.application;
 
 import com.yahoo.config.model.api.Reindexing;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.DocumentDatabase;
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
 import static java.util.stream.Collectors.toUnmodifiableMap;
+import static java.util.stream.Collectors.toUnmodifiableSet;
 
 /**
  * Pending and ready reindexing per document type. Each document type can have either a pending or a ready reindexing.
@@ -38,6 +50,40 @@ public class ApplicationReindexing implements Reindexing {
         return new ApplicationReindexing(true, new Status(now), Map.of());
     }
 
+    /** Returns the set of document types in each content cluster, in the given application */
+    public static Map<String, Set<String>> documentTypes(Application application) {
+        Map<String, ContentCluster> contentClusters = ((VespaModel) application.getModel()).getContentClusters();
+        return contentClusters.entrySet().stream()
+                              .collect(toMap(cluster -> cluster.getKey(),
+                                             cluster -> cluster.getValue().getDocumentDefinitions().keySet()));
+    }
+
+    /** Returns the set of document types in each cluster, in the given application, that have an index for one of more fields. */
+    public static Map<String, Set<String>> documentTypesWithIndex(Application application) {
+        Map<String, ContentCluster> contentClusters = ((VespaModel) application.getModel()).getContentClusters();
+        return contentClusters.entrySet().stream()
+                              .collect(toUnmodifiableMap(cluster -> cluster.getKey(),
+                                                         cluster -> documentTypesWithIndex(cluster.getValue())));
+    }
+
+    private static Set<String> documentTypesWithIndex(ContentCluster content) {
+        Set<String> typesWithIndexMode = content.getSearch().getDocumentTypesWithIndexedCluster().stream()
+                                                .map(type -> type.getFullName().getName())
+                                                .collect(toSet());
+
+        Set<String> typesWithIndexedFields = content.getSearch().getIndexed() == null
+                                             ? Set.of()
+                                             : content.getSearch().getIndexed().getDocumentDbs().stream()
+                                                      .filter(database -> database.getDerivedConfiguration()
+                                                                                  .getSearch()
+                                                                                  .allConcreteFields()
+                                                                                  .stream().anyMatch(SDField::doesIndexing))
+                                                      .map(database -> database.getInputDocType())
+                                                      .collect(toSet());
+
+        return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains).collect(toUnmodifiableSet());
+    }
+
     /** Returns a copy of this with reindexing for the whole application ready at the given instant. */
     public ApplicationReindexing withReady(Instant readyAt) {
         return new ApplicationReindexing(enabled,
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 7ea53a66697..401823aa6cd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -18,6 +18,7 @@ import com.yahoo.jdisc.application.UriPattern;
 import com.yahoo.slime.Cursor;
 import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.application.ClusterReindexing;
 import com.yahoo.vespa.config.server.http.ContentHandler;
@@ -38,6 +39,9 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.StringJoiner;
+import java.util.TreeMap;
+import java.util.TreeSet;
 import java.util.stream.Stream;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -227,29 +231,46 @@ public class ApplicationHandler extends HttpHandler {
     }
 
     private HttpResponse triggerReindexing(HttpRequest request, ApplicationId applicationId) {
+        Application application = applicationRepository.getActiveApplicationSet(applicationId)
+                .orElseThrow(() -> new NotFoundException(applicationId + " not found"))
+                .getForVersionOrLatest(Optional.empty(), applicationRepository.clock().instant());
+        Map<String, Set<String>> documentTypes = ApplicationReindexing.documentTypes(application);
+        Map<String, Set<String>> indexedDocumentTypes = ApplicationReindexing.documentTypesWithIndex(application);
+
+        boolean indexedOnly = request.getBooleanProperty("indexedOnly");
         Set<String> clusters = StringUtilities.split(request.getProperty("clusterId"));
         Set<String> types = StringUtilities.split(request.getProperty("documentType"));
+
+        Map<String, Set<String>> reindexed = new TreeMap<>();
         Instant now = applicationRepository.clock().instant();
         applicationRepository.modifyReindexing(applicationId, reindexing -> {
-            if (clusters.isEmpty())
-                reindexing = reindexing.withReady(now);
-            else
-                for (String cluster : clusters)
-                    if (types.isEmpty())
-                        reindexing = reindexing.withReady(cluster, now);
-                    else
-                        for (String type : types)
-                            reindexing = reindexing.withReady(cluster, type, now);
+            for (String cluster : clusters.isEmpty() ? documentTypes.keySet() : clusters) {
+                if ( ! documentTypes.containsKey(cluster))
+                    throw new IllegalArgumentException("No content cluster '" + cluster + "' in application — only: " +
+                                                       String.join(", ", documentTypes.keySet()));
+
+                for (String type : types.isEmpty() ? documentTypes.get(cluster) : types) {
+                    if ( ! documentTypes.get(cluster).contains(type))
+                        throw new IllegalArgumentException("No document type '" + type + "' in cluster '" + cluster + "' — only: " +
+                                                           String.join(", ", documentTypes.get(cluster)));
+
+                    if ( ! indexedOnly || indexedDocumentTypes.get(cluster).contains(type)) {
+                        reindexing = reindexing.withReady(cluster, type, now);
+                        reindexed.computeIfAbsent(cluster, __ -> new TreeSet<>()).add(type);
+                    }
+                }
+            }
             return reindexing;
         });
 
-        String message = "Reindexing " +
-                         (clusters.isEmpty() ? ""
-                                             : (types.isEmpty() ? ""
-                                                                : "document types " + String.join(", ", types) + " in ") +
-                                               "clusters " + String.join(", ", clusters) + " of ") +
-                         "application " + applicationId;
-        return createMessageResponse(message);
+        return createMessageResponse(reindexed.entrySet().stream()
+                                              .filter(cluster -> ! cluster.getValue().isEmpty())
+                                              .map(cluster -> "[" + String.join(", ", cluster.getValue()) + "] in '" + cluster.getKey() + "'")
+                                              .reduce(new StringJoiner(", ", "Reindexing document types ", " of application " + applicationId)
+                                                              .setEmptyValue("Not reindexing any document types of application " + applicationId),
+                                                      StringJoiner::add,
+                                                      StringJoiner::merge)
+                                              .toString());
     }
 
     private HttpResponse getReindexingStatus(ApplicationId applicationId) {
@@ -452,8 +473,6 @@ public class ApplicationHandler extends HttpHandler {
         ReindexingResponse(ApplicationReindexing reindexing, Map<String, ClusterReindexing> clusters) {
             super(Response.Status.OK);
             object.setBool("enabled", reindexing.enabled());
-            setStatus(object.setObject("status"), reindexing.common());
-
             Cursor clustersObject = object.setObject("clusters");
             Stream<String> clusterNames = Stream.concat(clusters.keySet().stream(), reindexing.clusters().keySet().stream());
             clusterNames.sorted()
@@ -464,8 +483,6 @@ public class ApplicationHandler extends HttpHandler {
 
                             Map<String, Cursor> statuses = new HashMap<>();
                             if (reindexing.clusters().containsKey(clusterName)) {
-                                setStatus(clusterObject.setObject("status"), reindexing.clusters().get(clusterName).common());
-
                                 reindexing.clusters().get(clusterName).pending().entrySet().stream().sorted(comparingByKey())
                                           .forEach(pending -> pendingObject.setLong(pending.getKey(), pending.getValue()));
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
index a8290b55874..34e4a5becfb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
@@ -11,8 +11,6 @@ import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.flags.FlagSource;
-import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Clock;
@@ -27,7 +25,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
 /**
  * Watches pending reindexing, and sets these to ready when config convergence is observed.
@@ -101,7 +98,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
     }
 
     static ApplicationReindexing withOnlyCurrentData(ApplicationReindexing reindexing, Application application) {
-        return withOnlyCurrentData(reindexing, clusterDocumentTypes(application));
+        return withOnlyCurrentData(reindexing, ApplicationReindexing.documentTypes(application));
     }
 
     static ApplicationReindexing withOnlyCurrentData(ApplicationReindexing reindexing, Map<String, ? extends Collection<String>> clusterDocumentTypes) {
@@ -122,11 +119,4 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
         return reindexing;
     }
 
-    static Map<String, Collection<String>> clusterDocumentTypes(Application application) {
-        Map<String, ContentCluster> contentClusters = ((VespaModel) application.getModel()).getContentClusters();
-        return contentClusters.entrySet().stream()
-                .collect(Collectors.toMap(cluster -> cluster.getKey(),
-                                          cluster -> cluster.getValue().getDocumentDefinitions().keySet()));
-    }
-
 }
diff --git a/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml b/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml
new file mode 100644
index 00000000000..f4256c9fc81
--- /dev/null
+++ b/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<hosts>
+  <host name="mytesthost">
+    <alias>node1</alias>
+  </host>
+</hosts>
diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd
new file mode 100644
index 00000000000..b66695b17df
--- /dev/null
+++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd
@@ -0,0 +1,14 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search bar {
+
+  field zoo type string {
+    indexing: input moo | index | summary
+  }
+
+  document bar {
+    field moo type string {
+      indexing: summary | attribute
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd
new file mode 100644
index 00000000000..f9f6aba766e
--- /dev/null
+++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd
@@ -0,0 +1,10 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search bax {
+
+  document bax {
+    field moo type string {
+      indexing: summary | attribute
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd
new file mode 100644
index 00000000000..58f0aa16fd0
--- /dev/null
+++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd
@@ -0,0 +1,10 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search baz {
+
+  document baz {
+    field moo type string {
+      indexing: summary | attribute
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/configserver/src/test/apps/app-with-multiple-clusters/services.xml b/configserver/src/test/apps/app-with-multiple-clusters/services.xml
new file mode 100644
index 00000000000..735bd04b2f9
--- /dev/null
+++ b/configserver/src/test/apps/app-with-multiple-clusters/services.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version="1.0">
+
+    <content id="foo" version="1.0">
+        <redundancy>2</redundancy>
+        <documents>
+            <document type="bar" mode="index"/>
+            <document type="baz" mode="streaming"/>
+            <document type="bax" mode="index"/>
+        </documents>
+        <nodes>
+            <node hostalias="node1" distribution-key="0"/>
+        </nodes>
+    </content>
+
+    <content id="boo" version="1.0">
+        <redundancy>2</redundancy>
+        <documents>
+            <document type="bar" mode="store-only"/>
+        </documents>
+        <nodes>
+            <node hostalias="node1" distribution-key="1"/>
+        </nodes>
+    </content>
+
+    <container version="1.0">
+        <nodes>
+            <node hostalias="node1" />
+        </nodes>
+    </container>
+
+</services>
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index f9b33791a36..60683b96f07 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -77,6 +77,7 @@ import static org.mockito.Mockito.when;
 public class ApplicationHandlerTest {
 
     private static final File testApp = new File("src/test/apps/app");
+    private static final File testAppMultipleClusters = new File("src/test/apps/app-with-multiple-clusters");
     private static final File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only");
 
     private final static TenantName mytenantName = TenantName.from("mytenant");
@@ -215,42 +216,55 @@ public class ApplicationHandlerTest {
         ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database();
         reindexing(applicationId, GET, "{\"error-code\": \"NOT_FOUND\", \"message\": \"Reindexing status not found for default.default\"}", 404);
 
-        applicationRepository.deploy(testApp, prepareParams(applicationId));
+        applicationRepository.deploy(testAppMultipleClusters, prepareParams(applicationId));
         ApplicationReindexing expected = ApplicationReindexing.ready(clock.instant());
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        reindex(applicationId, "", "{\"message\":\"Reindexing application default.default\"}");
-        expected = expected.withReady(clock.instant());
+        reindex(applicationId, "", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}");
+        expected = expected.withReady("boo", "bar", clock.instant())
+                           .withReady("foo", "bar", clock.instant())
+                           .withReady("foo", "baz", clock.instant())
+                           .withReady("foo", "bax", clock.instant());
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        expected = expected.withReady(clock.instant());
-        reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing application default.default\"}");
+        reindex(applicationId, "?indexedOnly=true", "{\"message\":\"Reindexing document types [bar] in 'foo' of application default.default\"}");
+        expected = expected.withReady("foo", "bar", clock.instant());
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        expected = expected.withReady(clock.instant());
-        reindex(applicationId, "?documentType=moo", "{\"message\":\"Reindexing application default.default\"}");
+        expected = expected.withReady("boo", "bar", clock.instant())
+                           .withReady("foo", "bar", clock.instant())
+                           .withReady("foo", "baz", clock.instant())
+                           .withReady("foo", "bax", clock.instant());
+        reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}");
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing clusters foo, boo of application default.default\"}");
-        expected = expected.withReady("foo", clock.instant())
-                           .withReady("boo", clock.instant());
+        expected = expected.withReady("boo", "bar", clock.instant())
+                           .withReady("foo", "bar", clock.instant());
+        reindex(applicationId, "?documentType=bar", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar] in 'foo' of application default.default\"}");
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        reindex(applicationId, "?clusterId=foo,boo&documentType=bar,baz", "{\"message\":\"Reindexing document types bar, baz in clusters foo, boo of application default.default\"}");
-        expected = expected.withReady("foo", "bar", clock.instant())
+        reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}");
+        expected = expected.withReady("boo", "bar", clock.instant())
+                           .withReady("foo", "bar", clock.instant())
                            .withReady("foo", "baz", clock.instant())
-                           .withReady("boo", "bar", clock.instant())
-                           .withReady("boo", "baz", clock.instant());
+                           .withReady("foo", "bax", clock.instant());
+        assertEquals(expected,
+                     database.readReindexingStatus(applicationId).orElseThrow());
+
+        clock.advance(Duration.ofSeconds(1));
+        reindex(applicationId, "?clusterId=foo&documentType=bar,baz", "{\"message\":\"Reindexing document types [bar, baz] in 'foo' of application default.default\"}");
+        expected = expected.withReady("foo", "bar", clock.instant())
+                           .withReady("foo", "baz", clock.instant());
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
@@ -269,35 +283,26 @@ public class ApplicationHandlerTest {
         long now = clock.instant().toEpochMilli();
         reindexing(applicationId, GET, "{" +
                                        "  \"enabled\": true," +
-                                       "  \"status\": {" +
-                                       "    \"readyMillis\": " + (now - 2000) +
-                                       "  }," +
                                        "  \"clusters\": {" +
                                        "    \"boo\": {" +
-                                       "      \"status\": {" +
-                                       "        \"readyMillis\": " + (now - 1000) +
-                                       "      }," +
                                        "      \"pending\": {" +
                                        "        \"bar\": 123" +
                                        "      }," +
                                        "      \"ready\": {" +
                                        "        \"bar\": {" +
-                                       "          \"readyMillis\": " + now +
+                                       "          \"readyMillis\": " + (now - 1000) +
                                        "        }," +
-                                       "        \"baz\": {" +
-                                       "          \"readyMillis\": " + now +
-                                       "        }" +
                                        "      }" +
                                        "    }," +
                                        "    \"foo\": {" +
-                                       "      \"status\": {" +
-                                       "        \"readyMillis\": " + (now - 1000) +
-                                       "      }," +
                                        "      \"pending\": {}," +
                                        "      \"ready\": {" +
                                        "        \"bar\": {" +
                                        "          \"readyMillis\": " + now +
                                        "        }," +
+                                       "        \"bax\": {" +
+                                       "          \"readyMillis\": " + (now - 1000) +
+                                       "        }," +
                                        "        \"baz\": {" +
                                        "          \"readyMillis\": " + now +
                                        "        }" +