From 1e57b906a0ed44a8d745030842a482f2347fe1b5 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 14 Jan 2021 20:54:02 +0100 Subject: Let all ApplicationReindexing users config-aware This allows checking for valid clusters and document types when manipulating status. This commit also updates responses in the application/v2/ API to reflect this --- .../server/application/ApplicationReindexing.java | 46 +++++++++++++++++ .../config/server/http/v2/ApplicationHandler.java | 57 +++++++++++++-------- .../server/maintenance/ReindexingMaintainer.java | 12 +---- .../test/apps/app-with-multiple-clusters/hosts.xml | 7 +++ .../apps/app-with-multiple-clusters/schemas/bar.sd | 14 +++++ .../apps/app-with-multiple-clusters/schemas/bax.sd | 10 ++++ .../apps/app-with-multiple-clusters/schemas/baz.sd | 10 ++++ .../apps/app-with-multiple-clusters/services.xml | 33 ++++++++++++ .../server/http/v2/ApplicationHandlerTest.java | 59 ++++++++++++---------- 9 files changed, 190 insertions(+), 58 deletions(-) create mode 100644 configserver/src/test/apps/app-with-multiple-clusters/hosts.xml create mode 100644 configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd create mode 100644 configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd create mode 100644 configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd create mode 100644 configserver/src/test/apps/app-with-multiple-clusters/services.xml (limited to 'configserver') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java index 2828b4c62e0..1736b23012d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java @@ -2,16 +2,28 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.model.api.Reindexing; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.DocumentDatabase; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Collection; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toUnmodifiableMap; +import static java.util.stream.Collectors.toUnmodifiableSet; /** * Pending and ready reindexing per document type. Each document type can have either a pending or a ready reindexing. @@ -38,6 +50,40 @@ public class ApplicationReindexing implements Reindexing { return new ApplicationReindexing(true, new Status(now), Map.of()); } + /** Returns the set of document types in each content cluster, in the given application */ + public static Map> documentTypes(Application application) { + Map contentClusters = ((VespaModel) application.getModel()).getContentClusters(); + return contentClusters.entrySet().stream() + .collect(toMap(cluster -> cluster.getKey(), + cluster -> cluster.getValue().getDocumentDefinitions().keySet())); + } + + /** Returns the set of document types in each cluster, in the given application, that have an index for one of more fields. */ + public static Map> documentTypesWithIndex(Application application) { + Map contentClusters = ((VespaModel) application.getModel()).getContentClusters(); + return contentClusters.entrySet().stream() + .collect(toUnmodifiableMap(cluster -> cluster.getKey(), + cluster -> documentTypesWithIndex(cluster.getValue()))); + } + + private static Set documentTypesWithIndex(ContentCluster content) { + Set typesWithIndexMode = content.getSearch().getDocumentTypesWithIndexedCluster().stream() + .map(type -> type.getFullName().getName()) + .collect(toSet()); + + Set typesWithIndexedFields = content.getSearch().getIndexed() == null + ? Set.of() + : content.getSearch().getIndexed().getDocumentDbs().stream() + .filter(database -> database.getDerivedConfiguration() + .getSearch() + .allConcreteFields() + .stream().anyMatch(SDField::doesIndexing)) + .map(database -> database.getInputDocType()) + .collect(toSet()); + + return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains).collect(toUnmodifiableSet()); + } + /** Returns a copy of this with reindexing for the whole application ready at the given instant. */ public ApplicationReindexing withReady(Instant readyAt) { return new ApplicationReindexing(enabled, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 7ea53a66697..401823aa6cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -18,6 +18,7 @@ import com.yahoo.jdisc.application.UriPattern; import com.yahoo.slime.Cursor; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationReindexing; import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.http.ContentHandler; @@ -38,6 +39,9 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.StringJoiner; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -227,29 +231,46 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse triggerReindexing(HttpRequest request, ApplicationId applicationId) { + Application application = applicationRepository.getActiveApplicationSet(applicationId) + .orElseThrow(() -> new NotFoundException(applicationId + " not found")) + .getForVersionOrLatest(Optional.empty(), applicationRepository.clock().instant()); + Map> documentTypes = ApplicationReindexing.documentTypes(application); + Map> indexedDocumentTypes = ApplicationReindexing.documentTypesWithIndex(application); + + boolean indexedOnly = request.getBooleanProperty("indexedOnly"); Set clusters = StringUtilities.split(request.getProperty("clusterId")); Set types = StringUtilities.split(request.getProperty("documentType")); + + Map> reindexed = new TreeMap<>(); Instant now = applicationRepository.clock().instant(); applicationRepository.modifyReindexing(applicationId, reindexing -> { - if (clusters.isEmpty()) - reindexing = reindexing.withReady(now); - else - for (String cluster : clusters) - if (types.isEmpty()) - reindexing = reindexing.withReady(cluster, now); - else - for (String type : types) - reindexing = reindexing.withReady(cluster, type, now); + for (String cluster : clusters.isEmpty() ? documentTypes.keySet() : clusters) { + if ( ! documentTypes.containsKey(cluster)) + throw new IllegalArgumentException("No content cluster '" + cluster + "' in application — only: " + + String.join(", ", documentTypes.keySet())); + + for (String type : types.isEmpty() ? documentTypes.get(cluster) : types) { + if ( ! documentTypes.get(cluster).contains(type)) + throw new IllegalArgumentException("No document type '" + type + "' in cluster '" + cluster + "' — only: " + + String.join(", ", documentTypes.get(cluster))); + + if ( ! indexedOnly || indexedDocumentTypes.get(cluster).contains(type)) { + reindexing = reindexing.withReady(cluster, type, now); + reindexed.computeIfAbsent(cluster, __ -> new TreeSet<>()).add(type); + } + } + } return reindexing; }); - String message = "Reindexing " + - (clusters.isEmpty() ? "" - : (types.isEmpty() ? "" - : "document types " + String.join(", ", types) + " in ") + - "clusters " + String.join(", ", clusters) + " of ") + - "application " + applicationId; - return createMessageResponse(message); + return createMessageResponse(reindexed.entrySet().stream() + .filter(cluster -> ! cluster.getValue().isEmpty()) + .map(cluster -> "[" + String.join(", ", cluster.getValue()) + "] in '" + cluster.getKey() + "'") + .reduce(new StringJoiner(", ", "Reindexing document types ", " of application " + applicationId) + .setEmptyValue("Not reindexing any document types of application " + applicationId), + StringJoiner::add, + StringJoiner::merge) + .toString()); } private HttpResponse getReindexingStatus(ApplicationId applicationId) { @@ -452,8 +473,6 @@ public class ApplicationHandler extends HttpHandler { ReindexingResponse(ApplicationReindexing reindexing, Map clusters) { super(Response.Status.OK); object.setBool("enabled", reindexing.enabled()); - setStatus(object.setObject("status"), reindexing.common()); - Cursor clustersObject = object.setObject("clusters"); Stream clusterNames = Stream.concat(clusters.keySet().stream(), reindexing.clusters().keySet().stream()); clusterNames.sorted() @@ -464,8 +483,6 @@ public class ApplicationHandler extends HttpHandler { Map statuses = new HashMap<>(); if (reindexing.clusters().containsKey(clusterName)) { - setStatus(clusterObject.setObject("status"), reindexing.clusters().get(clusterName).common()); - reindexing.clusters().get(clusterName).pending().entrySet().stream().sorted(comparingByKey()) .forEach(pending -> pendingObject.setLong(pending.getKey(), pending.getValue())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index a8290b55874..34e4a5becfb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -11,8 +11,6 @@ import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.yolean.Exceptions; import java.time.Clock; @@ -27,7 +25,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Watches pending reindexing, and sets these to ready when config convergence is observed. @@ -101,7 +98,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { } static ApplicationReindexing withOnlyCurrentData(ApplicationReindexing reindexing, Application application) { - return withOnlyCurrentData(reindexing, clusterDocumentTypes(application)); + return withOnlyCurrentData(reindexing, ApplicationReindexing.documentTypes(application)); } static ApplicationReindexing withOnlyCurrentData(ApplicationReindexing reindexing, Map> clusterDocumentTypes) { @@ -122,11 +119,4 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { return reindexing; } - static Map> clusterDocumentTypes(Application application) { - Map contentClusters = ((VespaModel) application.getModel()).getContentClusters(); - return contentClusters.entrySet().stream() - .collect(Collectors.toMap(cluster -> cluster.getKey(), - cluster -> cluster.getValue().getDocumentDefinitions().keySet())); - } - } diff --git a/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml b/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml new file mode 100644 index 00000000000..f4256c9fc81 --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml @@ -0,0 +1,7 @@ + + + + + node1 + + diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd new file mode 100644 index 00000000000..b66695b17df --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd @@ -0,0 +1,14 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search bar { + + field zoo type string { + indexing: input moo | index | summary + } + + document bar { + field moo type string { + indexing: summary | attribute + } + } + +} \ No newline at end of file diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd new file mode 100644 index 00000000000..f9f6aba766e --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search bax { + + document bax { + field moo type string { + indexing: summary | attribute + } + } + +} \ No newline at end of file diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd new file mode 100644 index 00000000000..58f0aa16fd0 --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search baz { + + document baz { + field moo type string { + indexing: summary | attribute + } + } + +} \ No newline at end of file diff --git a/configserver/src/test/apps/app-with-multiple-clusters/services.xml b/configserver/src/test/apps/app-with-multiple-clusters/services.xml new file mode 100644 index 00000000000..735bd04b2f9 --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/services.xml @@ -0,0 +1,33 @@ + + + + + + 2 + + + + + + + + + + + + 2 + + + + + + + + + + + + + + + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index f9b33791a36..60683b96f07 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -77,6 +77,7 @@ import static org.mockito.Mockito.when; public class ApplicationHandlerTest { private static final File testApp = new File("src/test/apps/app"); + private static final File testAppMultipleClusters = new File("src/test/apps/app-with-multiple-clusters"); private static final File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only"); private final static TenantName mytenantName = TenantName.from("mytenant"); @@ -215,42 +216,55 @@ public class ApplicationHandlerTest { ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database(); reindexing(applicationId, GET, "{\"error-code\": \"NOT_FOUND\", \"message\": \"Reindexing status not found for default.default\"}", 404); - applicationRepository.deploy(testApp, prepareParams(applicationId)); + applicationRepository.deploy(testAppMultipleClusters, prepareParams(applicationId)); ApplicationReindexing expected = ApplicationReindexing.ready(clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "", "{\"message\":\"Reindexing application default.default\"}"); - expected = expected.withReady(clock.instant()); + reindex(applicationId, "", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}"); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()) + .withReady("foo", "baz", clock.instant()) + .withReady("foo", "bax", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - expected = expected.withReady(clock.instant()); - reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing application default.default\"}"); + reindex(applicationId, "?indexedOnly=true", "{\"message\":\"Reindexing document types [bar] in 'foo' of application default.default\"}"); + expected = expected.withReady("foo", "bar", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - expected = expected.withReady(clock.instant()); - reindex(applicationId, "?documentType=moo", "{\"message\":\"Reindexing application default.default\"}"); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()) + .withReady("foo", "baz", clock.instant()) + .withReady("foo", "bax", clock.instant()); + reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}"); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing clusters foo, boo of application default.default\"}"); - expected = expected.withReady("foo", clock.instant()) - .withReady("boo", clock.instant()); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()); + reindex(applicationId, "?documentType=bar", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar] in 'foo' of application default.default\"}"); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "?clusterId=foo,boo&documentType=bar,baz", "{\"message\":\"Reindexing document types bar, baz in clusters foo, boo of application default.default\"}"); - expected = expected.withReady("foo", "bar", clock.instant()) + reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}"); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()) .withReady("foo", "baz", clock.instant()) - .withReady("boo", "bar", clock.instant()) - .withReady("boo", "baz", clock.instant()); + .withReady("foo", "bax", clock.instant()); + assertEquals(expected, + database.readReindexingStatus(applicationId).orElseThrow()); + + clock.advance(Duration.ofSeconds(1)); + reindex(applicationId, "?clusterId=foo&documentType=bar,baz", "{\"message\":\"Reindexing document types [bar, baz] in 'foo' of application default.default\"}"); + expected = expected.withReady("foo", "bar", clock.instant()) + .withReady("foo", "baz", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); @@ -269,35 +283,26 @@ public class ApplicationHandlerTest { long now = clock.instant().toEpochMilli(); reindexing(applicationId, GET, "{" + " \"enabled\": true," + - " \"status\": {" + - " \"readyMillis\": " + (now - 2000) + - " }," + " \"clusters\": {" + " \"boo\": {" + - " \"status\": {" + - " \"readyMillis\": " + (now - 1000) + - " }," + " \"pending\": {" + " \"bar\": 123" + " }," + " \"ready\": {" + " \"bar\": {" + - " \"readyMillis\": " + now + + " \"readyMillis\": " + (now - 1000) + " }," + - " \"baz\": {" + - " \"readyMillis\": " + now + - " }" + " }" + " }," + " \"foo\": {" + - " \"status\": {" + - " \"readyMillis\": " + (now - 1000) + - " }," + " \"pending\": {}," + " \"ready\": {" + " \"bar\": {" + " \"readyMillis\": " + now + " }," + + " \"bax\": {" + + " \"readyMillis\": " + (now - 1000) + + " }," + " \"baz\": {" + " \"readyMillis\": " + now + " }" + -- cgit v1.2.3