diff options
35 files changed, 324 insertions, 213 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index 6481c4cb2c7..36479c7504a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -27,18 +27,7 @@ public interface Model { * @param configKey the key to resolve * @param configDefinition the config definition to use for the schema */ - @Deprecated // TODO: Return after December 2020 - ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition configDefinition); - - /** - * Resolves config for a given key and config definition - * - * @param configKey the key to resolve - * @param configDefinition the config definition to use for the schema - */ - default ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition configDefinition) { - return null; // TODO: Remove this default implementation after December 2020 - } + ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition configDefinition); /** Produces a set of the valid config keys for this model. */ Set<ConfigKey<?>> allConfigsProduced(); diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index df9f72b2182..38d831a0b28 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -34,15 +34,14 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public static final class Name { - // TODO: privatize - final String name; - final int id; + private final String name; + private final int id; public Name(String name) { - this(name.hashCode(),name); + this(name.hashCode(), name); } - public Name(int id,String name) { + public Name(int id, String name) { this.id = id; this.name = name; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 9f9c5def406..acb4f58655d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -404,25 +404,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** * Resolve config for a given key and config definition * - * @param configKey The key to resolve. - * @param targetDef The config definition to use for the schema - * @return The payload as a list of strings - */ - @Deprecated // TODO: Remove after December 2020 - @Override - public ConfigPayload getConfig(ConfigKey<?> configKey, com.yahoo.vespa.config.buildergen.ConfigDefinition targetDef) { - Objects.requireNonNull(targetDef, "config definition cannot be null"); - - ConfigInstance.Builder builder = resolveToBuilder(configKey); - log.log(Level.FINE, () -> "Found builder for " + configKey); - InnerCNode innerCNode = targetDef.getCNode(); - ConfigPayload payload = getConfigFromBuilder(builder, innerCNode); - return (innerCNode != null) ? payload.applyDefaultsFromDef(innerCNode) : payload; - } - - /** - * Resolve config for a given key and config definition - * * @param configKey the key to resolve. * @param targetDef the config definition to use for the schema * @return the resolved config instance diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java index 58cea8c23e5..2fbf4359121 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -32,23 +32,19 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { for (Map.Entry<String, ContentCluster> currentEntry : current.getContentClusters().entrySet()) { ContentCluster nextCluster = next.getContentClusters().get(currentEntry.getKey()); if (nextCluster != null && nextCluster.getSearch().hasIndexedCluster()) { - result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, overrides, now)); + result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster)); } } return result; } private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, - ContentCluster nextCluster, - ValidationOverrides overrides, - Instant now) { - return validateDocumentDatabases(currentCluster, nextCluster, overrides, now); + ContentCluster nextCluster) { + return validateDocumentDatabases(currentCluster, nextCluster); } private static List<ConfigChangeAction> validateDocumentDatabases(ContentCluster currentCluster, - ContentCluster nextCluster, - ValidationOverrides overrides, - Instant now) { + ContentCluster nextCluster) { List<ConfigChangeAction> result = new ArrayList<>(); for (DocumentDatabase currentDb : getDocumentDbs(currentCluster.getSearch())) { String docTypeName = currentDb.getName(); @@ -56,7 +52,7 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { filter(db -> db.getName().equals(docTypeName)).findFirst(); if (nextDb.isPresent()) { result.addAll(validateDocumentDatabase(currentCluster, nextCluster, docTypeName, - currentDb, nextDb.get(), overrides, now)); + currentDb, nextDb.get())); } } return result; @@ -66,13 +62,11 @@ public class IndexedSearchClusterChangeValidator implements ChangeValidator { ContentCluster nextCluster, String docTypeName, DocumentDatabase currentDb, - DocumentDatabase nextDb, - ValidationOverrides overrides, - Instant now) { + DocumentDatabase nextDb) { NewDocumentType currentDocType = currentCluster.getDocumentDefinitions().get(docTypeName); NewDocumentType nextDocType = nextCluster.getDocumentDefinitions().get(docTypeName); List<VespaConfigChangeAction> result = - new DocumentDatabaseChangeValidator(currentCluster.id(), currentDb, currentDocType, nextDb, nextDocType).validate(overrides, now); + new DocumentDatabaseChangeValidator(currentCluster.id(), currentDb, currentDocType, nextDb, nextDocType).validate(); return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getIndexed()), docTypeName); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java index 385a678d452..b83c345efd3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -35,13 +35,12 @@ public class IndexingModeChangeValidator implements ChangeValidator { for (Map.Entry<String, ContentCluster> currentEntry : currentModel.getContentClusters().entrySet()) { ContentCluster nextCluster = nextModel.getContentClusters().get(currentEntry.getKey()); if (nextCluster == null) continue; - actions.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, overrides, now)); + actions.addAll(validateContentCluster(currentEntry.getValue(), nextCluster)); } return actions; } - private static List<ConfigChangeAction> validateContentCluster( - ContentCluster currentCluster, ContentCluster nextCluster, ValidationOverrides overrides, Instant now) { + private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster) { List<ConfigChangeAction> actions = new ArrayList<>(); ContentSearchCluster currentSearchCluster = currentCluster.getSearch(); ContentSearchCluster nextSearchCluster = nextCluster.getSearch(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java index ce435a4c157..5596169958a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -36,12 +36,12 @@ public class DocumentDatabaseChangeValidator { this.nextDocType = nextDocType; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(validateAttributeChanges()); - result.addAll(validateStructFieldAttributeChanges(overrides, now)); - result.addAll(validateIndexingScriptChanges(overrides, now)); - result.addAll(validateDocumentTypeChanges(overrides, now)); + result.addAll(validateStructFieldAttributeChanges()); + result.addAll(validateIndexingScriptChanges()); + result.addAll(validateDocumentTypeChanges()); return result; } @@ -54,23 +54,23 @@ public class DocumentDatabaseChangeValidator { .validate(); } - private List<VespaConfigChangeAction> validateStructFieldAttributeChanges(ValidationOverrides overrides, Instant now) { + private List<VespaConfigChangeAction> validateStructFieldAttributeChanges() { return new StructFieldAttributeChangeValidator(id, currentDocType, currentDatabase.getDerivedConfiguration().getAttributeFields(), nextDocType, nextDatabase.getDerivedConfiguration().getAttributeFields()) - .validate(overrides, now); + .validate(); } - private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides, Instant now) { + private List<VespaConfigChangeAction> validateIndexingScriptChanges() { return new IndexingScriptChangeValidator(id, currentDatabase.getDerivedConfiguration().getSearch(), nextDatabase.getDerivedConfiguration().getSearch()) - .validate(overrides, now); + .validate(); } - private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides, Instant now) { + private List<VespaConfigChangeAction> validateDocumentTypeChanges() { return new DocumentTypeChangeValidator(id, currentDocType, nextDocType) .validate(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index e3f3abf0747..91e370211f1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -35,7 +35,7 @@ public class IndexingScriptChangeValidator { this.nextSearch = nextSearch; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); for (ImmutableSDField nextField : nextSearch.allConcreteFields()) { String fieldName = nextField.getName(); @@ -68,8 +68,8 @@ public class IndexingScriptChangeValidator { return removeOutputExpressions(currentScript).equals(removeOutputExpressions(nextScript)); } - private static ScriptExpression removeOutputExpressions(ScriptExpression script) { - return (ScriptExpression) new OutputExpressionRemover().convert(script); + private static Expression removeOutputExpressions(ScriptExpression script) { + return new OutputExpressionRemover().convert(script); } private static class OutputExpressionRemover extends ExpressionConverter { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java index e8b2d593de6..a86277dded8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java @@ -52,20 +52,19 @@ public class StructFieldAttributeChangeValidator { this.nextAttributes = nextAttributes; } - public List<VespaConfigChangeAction> validate(ValidationOverrides overrides, Instant now) { + public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList(); for (Field currentField : currentDocType.getAllFields()) { Field nextField = nextDocType.getField(currentField.getName()); if (nextField != null) { result.addAll(validateAddAttributeAspect(new Context(currentField, currentAttributes), - new Context(nextField, nextAttributes), - overrides, now)); + new Context(nextField, nextAttributes))); } } return result; } - private List<VespaConfigChangeAction> validateAddAttributeAspect(Context current, Context next, ValidationOverrides overrides, Instant now) { + private List<VespaConfigChangeAction> validateAddAttributeAspect(Context current, Context next) { return next.structFieldAttributes.stream() .filter(nextAttr -> current.hasFieldForStructFieldAttribute(nextAttr) && !current.hasStructFieldAttribute(nextAttr)) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index 1f64d41e371..27a32f3e754 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -31,7 +31,7 @@ public class DocumentDatabaseChangeValidatorTest { @Override public List<VespaConfigChangeAction> validate() { - return validator.validate(ValidationOverrides.empty, Instant.now()); + return validator.validate(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 2e1ec53f886..20f5a9c841c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -29,7 +29,7 @@ public class IndexingScriptChangeValidatorTest { @Override public List<VespaConfigChangeAction> validate() { - return validator.validate(ValidationOverrides.empty, Instant.now()); + return validator.validate(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java index 0bc4ecbfdfd..d37ab8be9a2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidatorTestCase.java @@ -35,7 +35,7 @@ public class StructFieldAttributeChangeValidatorTestCase { @Override public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); - result.addAll(structFieldAttributeValidator.validate(ValidationOverrides.empty, Instant.now())); + result.addAll(structFieldAttributeValidator.validate()); result.addAll(docTypeValidator.validate()); return result; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index e77592dc011..57eac2a1a7b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -135,12 +135,7 @@ public class Application implements ModelResult { boolean applyOnRestart = false; try { builder = model.getConfigInstance(configKey, def); - if (builder == null) { // TODO: Remove this condition after December 2020 - payload = model.getConfig(configKey, def); - if (def.getCNode() != null) - payload.applyDefaultsFromDef(def.getCNode()); - } - else if (builder instanceof GenericConfig.GenericConfigBuilder) { + if (builder instanceof GenericConfig.GenericConfigBuilder) { payload = ((GenericConfig.GenericConfigBuilder) builder).getPayload(); applyOnRestart = builder.getApplyOnRestart(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java index cb81bed155e..1736b23012d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java @@ -2,17 +2,28 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.model.api.Reindexing; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.DocumentDatabase; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Collection; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toUnmodifiableMap; +import static java.util.stream.Collectors.toUnmodifiableSet; /** * Pending and ready reindexing per document type. Each document type can have either a pending or a ready reindexing. @@ -39,6 +50,40 @@ public class ApplicationReindexing implements Reindexing { return new ApplicationReindexing(true, new Status(now), Map.of()); } + /** Returns the set of document types in each content cluster, in the given application */ + public static Map<String, Set<String>> documentTypes(Application application) { + Map<String, ContentCluster> contentClusters = ((VespaModel) application.getModel()).getContentClusters(); + return contentClusters.entrySet().stream() + .collect(toMap(cluster -> cluster.getKey(), + cluster -> cluster.getValue().getDocumentDefinitions().keySet())); + } + + /** Returns the set of document types in each cluster, in the given application, that have an index for one of more fields. */ + public static Map<String, Set<String>> documentTypesWithIndex(Application application) { + Map<String, ContentCluster> contentClusters = ((VespaModel) application.getModel()).getContentClusters(); + return contentClusters.entrySet().stream() + .collect(toUnmodifiableMap(cluster -> cluster.getKey(), + cluster -> documentTypesWithIndex(cluster.getValue()))); + } + + private static Set<String> documentTypesWithIndex(ContentCluster content) { + Set<String> typesWithIndexMode = content.getSearch().getDocumentTypesWithIndexedCluster().stream() + .map(type -> type.getFullName().getName()) + .collect(toSet()); + + Set<String> typesWithIndexedFields = content.getSearch().getIndexed() == null + ? Set.of() + : content.getSearch().getIndexed().getDocumentDbs().stream() + .filter(database -> database.getDerivedConfiguration() + .getSearch() + .allConcreteFields() + .stream().anyMatch(SDField::doesIndexing)) + .map(database -> database.getInputDocType()) + .collect(toSet()); + + return typesWithIndexMode.stream().filter(typesWithIndexedFields::contains).collect(toUnmodifiableSet()); + } + /** Returns a copy of this with reindexing for the whole application ready at the given instant. */ public ApplicationReindexing withReady(Instant readyAt) { return new ApplicationReindexing(enabled, @@ -79,12 +124,32 @@ public class ApplicationReindexing implements Reindexing { /** Returns a copy of this with no pending reindexing for the given document type. */ public ApplicationReindexing withoutPending(String cluster, String documentType) { Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common)); + if (current == null) + return this; + Cluster modified = new Cluster(current.common, without(documentType, current.pending), current.ready); return new ApplicationReindexing(enabled, common, with(cluster, modified, clusters)); } + /** Returns a copy of this without the given cluster. */ + public ApplicationReindexing without(String cluster) { + return new ApplicationReindexing(enabled, common, without(cluster, clusters)); + } + + /** Returns a copy of this without the given document type in the given cluster. */ + public ApplicationReindexing without(String cluster, String documentType) { + Cluster current = clusters.get(cluster); + if (current == null) + return this; + + Cluster modified = new Cluster(current.common, + current.pending, + without(documentType, current.ready)); + return new ApplicationReindexing(enabled, common, with(cluster, modified, clusters)); + } + /** Returns a copy of this with the enabled-state set to the given value. */ public ApplicationReindexing enabled(boolean enabled) { return new ApplicationReindexing(enabled, common, clusters); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 7ea53a66697..401823aa6cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -18,6 +18,7 @@ import com.yahoo.jdisc.application.UriPattern; import com.yahoo.slime.Cursor; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationReindexing; import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.http.ContentHandler; @@ -38,6 +39,9 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.StringJoiner; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; @@ -227,29 +231,46 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse triggerReindexing(HttpRequest request, ApplicationId applicationId) { + Application application = applicationRepository.getActiveApplicationSet(applicationId) + .orElseThrow(() -> new NotFoundException(applicationId + " not found")) + .getForVersionOrLatest(Optional.empty(), applicationRepository.clock().instant()); + Map<String, Set<String>> documentTypes = ApplicationReindexing.documentTypes(application); + Map<String, Set<String>> indexedDocumentTypes = ApplicationReindexing.documentTypesWithIndex(application); + + boolean indexedOnly = request.getBooleanProperty("indexedOnly"); Set<String> clusters = StringUtilities.split(request.getProperty("clusterId")); Set<String> types = StringUtilities.split(request.getProperty("documentType")); + + Map<String, Set<String>> reindexed = new TreeMap<>(); Instant now = applicationRepository.clock().instant(); applicationRepository.modifyReindexing(applicationId, reindexing -> { - if (clusters.isEmpty()) - reindexing = reindexing.withReady(now); - else - for (String cluster : clusters) - if (types.isEmpty()) - reindexing = reindexing.withReady(cluster, now); - else - for (String type : types) - reindexing = reindexing.withReady(cluster, type, now); + for (String cluster : clusters.isEmpty() ? documentTypes.keySet() : clusters) { + if ( ! documentTypes.containsKey(cluster)) + throw new IllegalArgumentException("No content cluster '" + cluster + "' in application — only: " + + String.join(", ", documentTypes.keySet())); + + for (String type : types.isEmpty() ? documentTypes.get(cluster) : types) { + if ( ! documentTypes.get(cluster).contains(type)) + throw new IllegalArgumentException("No document type '" + type + "' in cluster '" + cluster + "' — only: " + + String.join(", ", documentTypes.get(cluster))); + + if ( ! indexedOnly || indexedDocumentTypes.get(cluster).contains(type)) { + reindexing = reindexing.withReady(cluster, type, now); + reindexed.computeIfAbsent(cluster, __ -> new TreeSet<>()).add(type); + } + } + } return reindexing; }); - String message = "Reindexing " + - (clusters.isEmpty() ? "" - : (types.isEmpty() ? "" - : "document types " + String.join(", ", types) + " in ") + - "clusters " + String.join(", ", clusters) + " of ") + - "application " + applicationId; - return createMessageResponse(message); + return createMessageResponse(reindexed.entrySet().stream() + .filter(cluster -> ! cluster.getValue().isEmpty()) + .map(cluster -> "[" + String.join(", ", cluster.getValue()) + "] in '" + cluster.getKey() + "'") + .reduce(new StringJoiner(", ", "Reindexing document types ", " of application " + applicationId) + .setEmptyValue("Not reindexing any document types of application " + applicationId), + StringJoiner::add, + StringJoiner::merge) + .toString()); } private HttpResponse getReindexingStatus(ApplicationId applicationId) { @@ -452,8 +473,6 @@ public class ApplicationHandler extends HttpHandler { ReindexingResponse(ApplicationReindexing reindexing, Map<String, ClusterReindexing> clusters) { super(Response.Status.OK); object.setBool("enabled", reindexing.enabled()); - setStatus(object.setObject("status"), reindexing.common()); - Cursor clustersObject = object.setObject("clusters"); Stream<String> clusterNames = Stream.concat(clusters.keySet().stream(), reindexing.clusters().keySet().stream()); clusterNames.sorted() @@ -464,8 +483,6 @@ public class ApplicationHandler extends HttpHandler { Map<String, Cursor> statuses = new HashMap<>(); if (reindexing.clusters().containsKey(clusterName)) { - setStatus(clusterObject.setObject("status"), reindexing.clusters().get(clusterName).common()); - reindexing.clusters().get(clusterName).pending().entrySet().stream().sorted(comparingByKey()) .forEach(pending -> pendingObject.setLong(pending.getKey(), pending.getValue())); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index 22eb95261bd..34e4a5becfb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationReindexing; +import com.yahoo.vespa.config.server.application.ApplicationReindexing.Cluster; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.curator.Curator; @@ -15,7 +16,9 @@ import com.yahoo.yolean.Exceptions; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collection; import java.util.Comparator; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -25,6 +28,7 @@ import java.util.logging.Logger; /** * Watches pending reindexing, and sets these to ready when config convergence is observed. + * Also removes data for clusters or document types which no longer exist. * * @author jonmv */ @@ -55,8 +59,11 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { .map(application -> application.getForVersionOrLatest(Optional.empty(), clock.instant())) .ifPresent(application -> { try { - applicationRepository.modifyReindexing(id, reindexing -> - withNewReady(reindexing, lazyGeneration(application), clock.instant())); + applicationRepository.modifyReindexing(id, reindexing -> { + reindexing = withNewReady(reindexing, lazyGeneration(application), clock.instant()); + reindexing = withOnlyCurrentData(reindexing, application); + return reindexing; + }); } catch (RuntimeException e) { log.log(Level.INFO, "Failed to update reindexing status for " + id + ": " + Exceptions.toMessageString(e)); @@ -90,4 +97,26 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { return reindexing; } + static ApplicationReindexing withOnlyCurrentData(ApplicationReindexing reindexing, Application application) { + return withOnlyCurrentData(reindexing, ApplicationReindexing.documentTypes(application)); + } + + static ApplicationReindexing withOnlyCurrentData(ApplicationReindexing reindexing, Map<String, ? extends Collection<String>> clusterDocumentTypes) { + for (String clusterId : reindexing.clusters().keySet()) { + if ( ! clusterDocumentTypes.containsKey(clusterId)) + reindexing = reindexing.without(clusterId); + else { + Cluster cluster = reindexing.clusters().get(clusterId); + Collection<String> documentTypes = clusterDocumentTypes.get(clusterId); + for (String pending : cluster.pending().keySet()) + if ( ! documentTypes.contains(pending)) + reindexing = reindexing.withoutPending(clusterId, pending); + for (String ready : cluster.ready().keySet()) + if ( ! documentTypes.contains(ready)) + reindexing = reindexing.without(clusterId, ready); + } + } + return reindexing; + } + } diff --git a/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml b/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml new file mode 100644 index 00000000000..f4256c9fc81 --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="mytesthost"> + <alias>node1</alias> + </host> +</hosts> diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd new file mode 100644 index 00000000000..b66695b17df --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bar.sd @@ -0,0 +1,14 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search bar { + + field zoo type string { + indexing: input moo | index | summary + } + + document bar { + field moo type string { + indexing: summary | attribute + } + } + +}
\ No newline at end of file diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd new file mode 100644 index 00000000000..f9f6aba766e --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/bax.sd @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search bax { + + document bax { + field moo type string { + indexing: summary | attribute + } + } + +}
\ No newline at end of file diff --git a/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd b/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd new file mode 100644 index 00000000000..58f0aa16fd0 --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/schemas/baz.sd @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search baz { + + document baz { + field moo type string { + indexing: summary | attribute + } + } + +}
\ No newline at end of file diff --git a/configserver/src/test/apps/app-with-multiple-clusters/services.xml b/configserver/src/test/apps/app-with-multiple-clusters/services.xml new file mode 100644 index 00000000000..735bd04b2f9 --- /dev/null +++ b/configserver/src/test/apps/app-with-multiple-clusters/services.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <content id="foo" version="1.0"> + <redundancy>2</redundancy> + <documents> + <document type="bar" mode="index"/> + <document type="baz" mode="streaming"/> + <document type="bax" mode="index"/> + </documents> + <nodes> + <node hostalias="node1" distribution-key="0"/> + </nodes> + </content> + + <content id="boo" version="1.0"> + <redundancy>2</redundancy> + <documents> + <document type="bar" mode="store-only"/> + </documents> + <nodes> + <node hostalias="node1" distribution-key="1"/> + </nodes> + </content> + + <container version="1.0"> + <nodes> + <node hostalias="node1" /> + </nodes> + </container> + +</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java index 88f7d352ad7..ed12f4dbbe1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java @@ -21,12 +21,6 @@ import java.util.Set; public class ModelStub implements Model { @Override - @SuppressWarnings("deprecation") - public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition targetDef) { - return null; - } - - @Override public ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition targetDef) { throw new UnsupportedOperationException(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 0288a551cd3..7e57eea74d6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -74,12 +74,6 @@ public class MockModel implements Model { } @Override - @SuppressWarnings("deprecation") - public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition targetDef) { - throw new UnsupportedOperationException(); - } - - @Override public ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition targetDef) { throw new UnsupportedOperationException(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index f9b33791a36..714526b24d4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -77,6 +77,7 @@ import static org.mockito.Mockito.when; public class ApplicationHandlerTest { private static final File testApp = new File("src/test/apps/app"); + private static final File testAppMultipleClusters = new File("src/test/apps/app-with-multiple-clusters"); private static final File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only"); private final static TenantName mytenantName = TenantName.from("mytenant"); @@ -215,42 +216,55 @@ public class ApplicationHandlerTest { ApplicationCuratorDatabase database = applicationRepository.getTenant(applicationId).getApplicationRepo().database(); reindexing(applicationId, GET, "{\"error-code\": \"NOT_FOUND\", \"message\": \"Reindexing status not found for default.default\"}", 404); - applicationRepository.deploy(testApp, prepareParams(applicationId)); + applicationRepository.deploy(testAppMultipleClusters, prepareParams(applicationId)); ApplicationReindexing expected = ApplicationReindexing.ready(clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "", "{\"message\":\"Reindexing application default.default\"}"); - expected = expected.withReady(clock.instant()); + reindex(applicationId, "", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}"); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()) + .withReady("foo", "baz", clock.instant()) + .withReady("foo", "bax", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - expected = expected.withReady(clock.instant()); - reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing application default.default\"}"); + reindex(applicationId, "?indexedOnly=true", "{\"message\":\"Reindexing document types [bar] in 'foo' of application default.default\"}"); + expected = expected.withReady("foo", "bar", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - expected = expected.withReady(clock.instant()); - reindex(applicationId, "?documentType=moo", "{\"message\":\"Reindexing application default.default\"}"); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()) + .withReady("foo", "baz", clock.instant()) + .withReady("foo", "bax", clock.instant()); + reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}"); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing clusters foo, boo of application default.default\"}"); - expected = expected.withReady("foo", clock.instant()) - .withReady("boo", clock.instant()); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()); + reindex(applicationId, "?documentType=bar", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar] in 'foo' of application default.default\"}"); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); clock.advance(Duration.ofSeconds(1)); - reindex(applicationId, "?clusterId=foo,boo&documentType=bar,baz", "{\"message\":\"Reindexing document types bar, baz in clusters foo, boo of application default.default\"}"); - expected = expected.withReady("foo", "bar", clock.instant()) + reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}"); + expected = expected.withReady("boo", "bar", clock.instant()) + .withReady("foo", "bar", clock.instant()) .withReady("foo", "baz", clock.instant()) - .withReady("boo", "bar", clock.instant()) - .withReady("boo", "baz", clock.instant()); + .withReady("foo", "bax", clock.instant()); + assertEquals(expected, + database.readReindexingStatus(applicationId).orElseThrow()); + + clock.advance(Duration.ofSeconds(1)); + reindex(applicationId, "?clusterId=foo&documentType=bar,baz", "{\"message\":\"Reindexing document types [bar, baz] in 'foo' of application default.default\"}"); + expected = expected.withReady("foo", "bar", clock.instant()) + .withReady("foo", "baz", clock.instant()); assertEquals(expected, database.readReindexingStatus(applicationId).orElseThrow()); @@ -269,35 +283,26 @@ public class ApplicationHandlerTest { long now = clock.instant().toEpochMilli(); reindexing(applicationId, GET, "{" + " \"enabled\": true," + - " \"status\": {" + - " \"readyMillis\": " + (now - 2000) + - " }," + " \"clusters\": {" + " \"boo\": {" + - " \"status\": {" + - " \"readyMillis\": " + (now - 1000) + - " }," + " \"pending\": {" + " \"bar\": 123" + " }," + " \"ready\": {" + " \"bar\": {" + - " \"readyMillis\": " + now + + " \"readyMillis\": " + (now - 1000) + " }," + - " \"baz\": {" + - " \"readyMillis\": " + now + - " }" + " }" + " }," + " \"foo\": {" + - " \"status\": {" + - " \"readyMillis\": " + (now - 1000) + - " }," + " \"pending\": {}," + " \"ready\": {" + " \"bar\": {" + " \"readyMillis\": " + now + " }," + + " \"bax\": {" + + " \"readyMillis\": " + (now - 1000) + + " }," + " \"baz\": {" + " \"readyMillis\": " + now + " }" + @@ -463,9 +468,6 @@ public class ApplicationHandlerTest { "moo", clusterReindexing))), "{\n" + " \"enabled\": true,\n" + - " \"status\": {\n" + - " \"readyMillis\": 113456\n" + - " },\n" + " \"clusters\": {\n" + " \"boo\": {\n" + " \"pending\": {},\n" + @@ -487,9 +489,6 @@ public class ApplicationHandlerTest { " \"bar\": 123\n" + " },\n" + " \"ready\": {},\n" + - " \"status\": {\n" + - " \"readyMillis\": 113456\n" + - " }\n" + " },\n" + " \"moo\": {\n" + " \"pending\": {},\n" + @@ -505,9 +504,6 @@ public class ApplicationHandlerTest { " \"bax\": {\n" + " \"startedMillis\": 123456\n" + " }\n" + - " },\n" + - " \"status\": {\n" + - " \"readyMillis\": 122456\n" + " }\n" + " }\n" + " }\n" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java index d75b91f45e3..d0a4cd59dbd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java @@ -5,8 +5,11 @@ import com.yahoo.vespa.config.server.application.ApplicationReindexing; import org.junit.Test; import java.time.Instant; +import java.util.List; +import java.util.Map; import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.withNewReady; +import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.withOnlyCurrentData; import static org.junit.Assert.assertEquals; /** @@ -46,4 +49,34 @@ public class ReindexingMaintainerTest { later); } + @Test + public void testGarbageRemoval() { + ApplicationReindexing reindexing = ApplicationReindexing.ready(Instant.EPOCH) + .withPending("one", "a", 10) + .withPending("two", "b", 20) + .withReady("one", "a", Instant.ofEpochMilli(3)) + .withReady("two", "b", Instant.ofEpochMilli(2)) + .withReady("two", "c", Instant.ofEpochMilli(3)); + + assertEquals(reindexing, + withOnlyCurrentData(reindexing, Map.of("one", List.of("a", "b", "c", "d"), + "two", List.of("b", "c"), + "three", List.of("a", "b")))); + + assertEquals(reindexing, + withOnlyCurrentData(reindexing, Map.of("one", List.of("a"), + "two", List.of("b", "c")))); + + assertEquals(ApplicationReindexing.ready(Instant.EPOCH) + .withPending("two", "b", 20) + .withReady("two", "b", Instant.ofEpochMilli(2)), + withOnlyCurrentData(reindexing, Map.of("two", List.of("a", "b")))); + + assertEquals(ApplicationReindexing.ready(Instant.EPOCH) + .withReady("one", Instant.EPOCH) + .withReady("two", "c", Instant.ofEpochMilli(3)), + withOnlyCurrentData(reindexing, Map.of("one", List.of("c"), + "two", List.of("c")))); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetrieverTest.java index b5cdfa8eda2..865d3b71b6e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/DeploymentMetricsRetrieverTest.java @@ -84,12 +84,6 @@ public class DeploymentMetricsRetrieverTest { } @Override - @SuppressWarnings("deprecation") - public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition targetDef) { - throw new UnsupportedOperationException(); - } - - @Override public ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition targetDef) { throw new UnsupportedOperationException(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java index 7fab01faf3d..5aada44b573 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ProtonMetricsRetrieverTest.java @@ -77,12 +77,6 @@ public class ProtonMetricsRetrieverTest { } @Override - @SuppressWarnings("deprecation") - public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition targetDef) { - throw new UnsupportedOperationException(); - } - - @Override public ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition targetDef) { throw new UnsupportedOperationException(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java index e143cdd6d9e..f94a91dc0c6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ApplicationReindexing.java @@ -6,8 +6,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import static java.util.Objects.requireNonNull; - /** * Reindexing status for a single Vespa application. * @@ -16,12 +14,10 @@ import static java.util.Objects.requireNonNull; public class ApplicationReindexing { private final boolean enabled; - private final Status common; private final Map<String, Cluster> clusters; - public ApplicationReindexing(boolean enabled, Status common, Map<String, Cluster> clusters) { + public ApplicationReindexing(boolean enabled, Map<String, Cluster> clusters) { this.enabled = enabled; - this.common = requireNonNull(common); this.clusters = Map.copyOf(clusters); } @@ -29,10 +25,6 @@ public class ApplicationReindexing { return enabled; } - public Status common() { - return common; - } - public Map<String, Cluster> clusters() { return clusters; } @@ -43,20 +35,18 @@ public class ApplicationReindexing { if (o == null || getClass() != o.getClass()) return false; ApplicationReindexing that = (ApplicationReindexing) o; return enabled == that.enabled && - common.equals(that.common) && clusters.equals(that.clusters); } @Override public int hashCode() { - return Objects.hash(enabled, common, clusters); + return Objects.hash(enabled, clusters); } @Override public String toString() { return "ApplicationReindexing{" + "enabled=" + enabled + - ", common=" + common + ", clusters=" + clusters + '}'; } @@ -64,20 +54,14 @@ public class ApplicationReindexing { public static class Cluster { - private final Optional<Status> common; private final Map<String, Long> pending; private final Map<String, Status> ready; - public Cluster(Status common, Map<String, Long> pending, Map<String, Status> ready) { - this.common = Optional.ofNullable(common); + public Cluster(Map<String, Long> pending, Map<String, Status> ready) { this.pending = Map.copyOf(pending); this.ready = Map.copyOf(ready); } - public Optional<Status> common() { - return common; - } - public Map<String, Long> pending() { return pending; } @@ -91,20 +75,18 @@ public class ApplicationReindexing { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Cluster cluster = (Cluster) o; - return common.equals(cluster.common) && - pending.equals(cluster.pending) && + return pending.equals(cluster.pending) && ready.equals(cluster.ready); } @Override public int hashCode() { - return Objects.hash(common, pending, ready); + return Objects.hash(pending, ready); } @Override public String toString() { return "Cluster{" + - "common=" + common + ", pending=" + pending + ", ready=" + ready + '}'; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 11940b30ac1..a90155d4e3e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -35,7 +35,7 @@ public interface ConfigServer { PreparedApplication deploy(DeploymentData deployment); - void reindex(DeploymentId deployment, List<String> clusterNames, List<String> documentTypes); + void reindex(DeploymentId deployment, List<String> clusterNames, List<String> documentTypes, boolean indexedOnly); Optional<ApplicationReindexing> getReindexing(DeploymentId deployment); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e071221dd05..8447353a869 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -189,8 +189,8 @@ public class ApplicationController { * if no documents types are given, reindexing is triggered for all given clusters; otherwise * reindexing is triggered for the cartesian product of the given clusters and document types. */ - public void reindex(ApplicationId id, ZoneId zoneId, List<String> clusterNames, List<String> documentTypes) { - configServer.reindex(new DeploymentId(id, zoneId), clusterNames, documentTypes); + public void reindex(ApplicationId id, ZoneId zoneId, List<String> clusterNames, List<String> documentTypes, boolean indexedOnly) { + configServer.reindex(new DeploymentId(id, zoneId), clusterNames, documentTypes, indexedOnly); } /** Returns the reindexing status for the given application in the given zone. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index f787c5d62e7..263a33cf266 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -49,7 +49,7 @@ public class ReindexingTriggerer extends ControllerMaintainer { for (Deployment deployment : deployments) if ( inWindowOfOpportunity(now, id, deployment.zone()) && reindexingIsReady(controller().applications().applicationReindexing(id, deployment.zone()), now)) - controller().applications().reindex(id, deployment.zone(), List.of(), List.of()); + controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true); }); return true; } @@ -74,11 +74,9 @@ public class ReindexingTriggerer extends ControllerMaintainer { } static boolean reindexingIsReady(ApplicationReindexing reindexing, Instant now) { - if (reindexing.clusters().values().stream().flatMap(cluster -> cluster.ready().values().stream()) - .anyMatch(status -> status.startedAt().isPresent() && status.endedAt().isEmpty())) - return false; - - return reindexing.common().readyAt().orElse(Instant.EPOCH).isBefore(now.minus(reindexingPeriod.dividedBy(2))); + return reindexing.clusters().values().stream().flatMap(cluster -> cluster.ready().values().stream()) + .allMatch(status -> status.readyAt().map(now.minus(reindexingPeriod.dividedBy(2))::isAfter).orElse(true) + && (status.startedAt().isEmpty() || status.endedAt().isPresent())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index be4372af526..8378b914fe4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1561,7 +1561,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .filter(type -> ! type.isBlank()) .collect(toUnmodifiableList()); - controller.applications().reindex(id, zone, clusterNames, documentTypes); + controller.applications().reindex(id, zone, clusterNames, documentTypes, request.getBooleanProperty("indexedOnly")); return new MessageResponse("Requested reindexing of " + id + " in " + zone + (clusterNames.isEmpty() ? "" : ", on clusters " + String.join(", ", clusterNames) + (documentTypes.isEmpty() ? "" : ", for types " + String.join(", ", documentTypes)))); @@ -1577,14 +1577,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Cursor root = slime.setObject(); root.setBool("enabled", reindexing.enabled()); - setStatus(root.setObject("status"), reindexing.common()); Cursor clustersArray = root.setArray("clusters"); reindexing.clusters().entrySet().stream().sorted(comparingByKey()) .forEach(cluster -> { Cursor clusterObject = clustersArray.addObject(); clusterObject.setString("name", cluster.getKey()); - cluster.getValue().common().ifPresent(common -> setStatus(clusterObject.setObject("status"), common)); Cursor pendingArray = clusterObject.setArray("pending"); cluster.getValue().pending().entrySet().stream().sorted(comparingByKey()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 7753570b72d..ee3c523a497 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -422,15 +422,13 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public void reindex(DeploymentId deployment, List<String> clusterNames, List<String> documentTypes) { } + public void reindex(DeploymentId deployment, List<String> clusterNames, List<String> documentTypes, boolean indexedOnly) { } @Override public Optional<ApplicationReindexing> getReindexing(DeploymentId deployment) { return Optional.of(new ApplicationReindexing(true, - new Status(Instant.ofEpochMilli(123)), Map.of("cluster", - new ApplicationReindexing.Cluster(new Status(Instant.ofEpochMilli(234)), - Map.of("type", 100L), + new ApplicationReindexing.Cluster(Map.of("type", 100L), Map.of("type", new Status(Instant.ofEpochMilli(345), Instant.ofEpochMilli(456), Instant.ofEpochMilli(567), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java index 848426b6581..3c22ee3f4c3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggererTest.java @@ -50,8 +50,7 @@ public class ReindexingTriggererTest { public void testReindexingIsReady() { Instant then = Instant.now(); ApplicationReindexing reindexing = new ApplicationReindexing(true, - new Status(then), - Map.of()); + Map.of("c", new Cluster(Map.of(), Map.of("d", new Status(then))))); Instant now = then; assertFalse("Should not be ready less than one half-period after last triggering", @@ -66,20 +65,16 @@ public class ReindexingTriggererTest { reindexingIsReady(reindexing, now)); reindexing = new ApplicationReindexing(true, - new Status(then), Map.of("cluster", - new Cluster(new Status(then), - Map.of(), + new Cluster(Map.of(), Map.of("type", new Status(then, then, null, null, null, null))))); assertFalse("Should not be ready when reindexing is already running", reindexingIsReady(reindexing, now)); reindexing = new ApplicationReindexing(true, - new Status(then), Map.of("cluster", - new Cluster(new Status(then), - Map.of("type", 123L), + new Cluster(Map.of("type", 123L), Map.of("type", new Status(then, then, now, null, null, null))))); assertTrue("Should be ready when reindexing is no longer running", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 84b6ec2b263..e5f11beb9a2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -615,7 +615,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET to get reindexing status tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/reindexing", GET) .userIdentity(USER_ID), - "{\"enabled\":true,\"status\":{\"readyAtMillis\":123},\"clusters\":[{\"name\":\"cluster\",\"status\":{\"readyAtMillis\":234},\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345,\"startedAtMillis\":456,\"endedAtMillis\":567,\"state\":\"failed\",\"message\":\"(＃｀д´)ﾉ\",\"progress\":0.1}]}]}"); + "{\"enabled\":true,\"clusters\":[{\"name\":\"cluster\",\"pending\":[{\"type\":\"type\",\"requiredGeneration\":100}],\"ready\":[{\"type\":\"type\",\"readyAtMillis\":345,\"startedAtMillis\":456,\"endedAtMillis\":567,\"state\":\"failed\",\"message\":\"(＃｀д´)ﾉ\",\"progress\":0.1}]}]}"); // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostsModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostsModel.java index 97b9313a06a..10820ccc508 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostsModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostsModel.java @@ -36,12 +36,6 @@ public class HostsModel implements Model { } @Override - @SuppressWarnings("deprecation") // yes, this is needed - public ConfigPayload getConfig(ConfigKey<?> configKey, ConfigDefinition configDefinition) { - throw new UnsupportedOperationException(); - } - - @Override public ConfigInstance.Builder getConfigInstance(ConfigKey<?> configKey, ConfigDefinition configDefinition) { throw new UnsupportedOperationException(); } |