diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-03-13 10:46:34 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2024-03-13 16:41:21 +0100 |
commit | 4528b9f815cf780fbda7ee0d041b8070a39d518d (patch) | |
tree | 4bbc1d7ea32a93b2c4097f6ad997acb2882307eb /config-model/src/main/java/com | |
parent | 37484d712d93d7daf6075455b266f18788d40690 (diff) |
Use a single searchcluster with both indexed and streaming docs.
Diffstat (limited to 'config-model/src/main/java/com')
13 files changed, 203 insertions, 207 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index b35918b3460..a7f66966538 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -28,7 +28,7 @@ import java.io.Writer; public class DerivedConfiguration { private final Schema schema; - private final boolean isStreaming; + private final SchemaInfo.IndexMode indexMode; private Summaries summaries; private Juniperrc juniperrc; private AttributeFields attributeFields; @@ -57,7 +57,8 @@ public class DerivedConfiguration { } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false); + this(new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), + schema, SchemaInfo.IndexMode.INDEX); } /** @@ -67,8 +68,8 @@ public class DerivedConfiguration { * argument is live. Which means that this object will be inconsistent if the given * schema is later modified. */ - public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { - this.isStreaming = isStreaming; + public DerivedConfiguration(DeployState deployState, Schema schema, SchemaInfo.IndexMode indexMode) { + this.indexMode = indexMode; try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -81,9 +82,9 @@ public class DerivedConfiguration { summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); - indexingScript = new IndexingScript(schema, isStreaming); - indexInfo = new IndexInfo(schema, isStreaming); - schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); + indexingScript = new IndexingScript(schema, isStreaming()); + indexInfo = new IndexInfo(schema, isStreaming()); + schemaInfo = new SchemaInfo(schema, indexMode, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } @@ -153,7 +154,7 @@ public class DerivedConfiguration { } public boolean isStreaming() { - return isStreaming; + return indexMode == SchemaInfo.IndexMode.STREAMING; } public Summaries getSummaries() { @@ -165,7 +166,11 @@ public class DerivedConfiguration { } public void getConfig(AttributesConfig.Builder builder) { - getConfig(builder, AttributeFields.FieldSet.ALL); + if (isStreaming()) { + getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } else { + getConfig(builder, AttributeFields.FieldSet.ALL); + } } public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index 19a045ac444..f996b2624db 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -42,13 +42,32 @@ public final class SchemaInfo extends Derived { private final Map<String, RankProfileInfo> rankProfiles; private final Summaries summaries; + private final IndexMode indexMode; - public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, Summaries summaries) { + public enum IndexMode {INDEX, STREAMING, STORE_ONLY} + + public SchemaInfo(Schema schema, String indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) { + this(schema, indexMode(indexMode), rankProfileRegistry, summaries); + } + public SchemaInfo(Schema schema, IndexMode indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) { this.schema = schema; this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); this.summaries = summaries; + this.indexMode = indexMode; } + private static IndexMode indexMode(String mode) { + if (mode == null) return IndexMode.INDEX; + return switch (mode) { + case "index" -> IndexMode.INDEX; + case "streaming" -> IndexMode.STREAMING; + case "store-only" -> IndexMode.STORE_ONLY; + default -> IndexMode.STORE_ONLY; + }; + } + + public IndexMode getIndexMode() { return indexMode; } + public String name() { return schema.getName(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java index 12f3f025996..7b7c92a43e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java @@ -28,10 +28,10 @@ public class ComplexFieldsWithStructFieldAttributesValidator implements Validato public void validate(Context context) { List<SearchCluster> searchClusters = context.model().getSearchClusters(); for (SearchCluster cluster : searchClusters) { - if (cluster.isStreaming()) continue; - for (SchemaInfo spec : cluster.schemas().values()) { - validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger()); + if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) { + validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger()); + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java index 3dbba081400..2d553c84f48 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.model.application.validation.Validation.Context; @@ -24,11 +25,10 @@ public class ComplexFieldsWithStructFieldIndexesValidator implements Validator { @Override public void validate(Context context) { for (var cluster : context.model().getSearchClusters()) { - if (cluster.isStreaming()) { - continue; - } for (var spec : cluster.schemas().values()) { - validateComplexFields(context, cluster.getClusterName(), spec.fullSchema()); + if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) { + validateComplexFields(context, cluster.getClusterName(), spec.fullSchema()); + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index 15d293e4abc..e1c64422653 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -23,10 +23,9 @@ public class NoPrefixForIndexes implements Validator { @Override public void validate(Context context) { for (SearchCluster cluster : context.model().getSearchClusters()) { - if (cluster instanceof IndexedSearchCluster) { - IndexedSearchCluster sc = (IndexedSearchCluster) cluster; - for (DocumentDatabase docDb : sc.getDocumentDbs()) { - DerivedConfiguration sdConfig = docDb.getDerivedConfiguration(); + for (DocumentDatabase docDb : cluster.getDocumentDbs()) { + DerivedConfiguration sdConfig = docDb.getDerivedConfiguration(); + if ( ! sdConfig.isStreaming() ) { Schema schema = sdConfig.getSchema(); for (ImmutableSDField field : schema.allConcreteFields()) { if (field.doesIndexing()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 03076b64d0c..736665b2184 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -69,7 +69,7 @@ public class RankSetupValidator implements Validator { String schemaDir = clusterDir + schemaName + "/"; writeConfigs(schemaDir, docDb); writeExtraVerifyRankSetupConfig(schemaDir, docDb); - if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, sc.isStreaming())) { + if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, docDb.getDerivedConfiguration().isStreaming())) { return; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 6e21adc4fe4..cd8094f8c12 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -31,15 +31,11 @@ public class SearchDataTypeValidator implements Validator { public void validate(Context context) { List<SearchCluster> clusters = context.model().getSearchClusters(); for (SearchCluster cluster : clusters) { - if (cluster.isStreaming()) { - continue; - } for (SchemaInfo spec : cluster.schemas().values()) { SDDocumentType docType = spec.fullSchema().getDocument(); - if (docType == null) { - continue; + if (docType != null && spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) { + validateDocument(context, cluster, spec.fullSchema(), docType); } - validateDocument(context, cluster, spec.fullSchema(), docType); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index c0ad55fc8f4..0e9d3bf7016 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -6,12 +6,14 @@ import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.TensorDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.MatchType; import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.SearchCluster; -import com.yahoo.vespa.model.search.StreamingSearchCluster; import java.util.List; import java.util.logging.Level; @@ -25,20 +27,21 @@ public class StreamingValidator implements Validator { public void validate(Context context) { List<SearchCluster> searchClusters = context.model().getSearchClusters(); for (SearchCluster cluster : searchClusters) { - if ( ! cluster.isStreaming()) continue; - - var streamingCluster = (StreamingSearchCluster)cluster; - warnStreamingAttributes(streamingCluster, context.deployState().getDeployLogger()); - warnStreamingGramMatching(streamingCluster, context.deployState().getDeployLogger()); - failStreamingDocumentReferences(context, streamingCluster); + for (SchemaInfo schemaInfo : cluster.schemas().values()) { + if (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING) { + var deployLogger = context.deployState().getDeployLogger(); + warnStreamingAttributes(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger); + warnStreamingGramMatching(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger); + failStreamingDocumentReferences(cluster.getClusterName(), cluster.getDocumentDB(schemaInfo.name()).getDerivedConfiguration(), context); + } + } } } - private static void warnStreamingGramMatching(StreamingSearchCluster sc, DeployLogger logger) { - for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { + private static void warnStreamingGramMatching(String cluster, Schema schema, DeployLogger logger) { + for (ImmutableSDField sd : schema.allConcreteFields()) { if (sd.getMatching().getType() == MatchType.GRAM) { - logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + - sc.getClusterName() + + logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster + "', schema '" + schema.getName() + "', SD field '" + sd.getName() + "': n-gram matching is not supported for streaming search."); } @@ -47,19 +50,16 @@ public class StreamingValidator implements Validator { /** * Warn if one or more attributes are defined in a streaming search cluster SD. - * - * @param sc a search cluster to be checked for attributes in streaming search - * @param logger a DeployLogger */ - private static void warnStreamingAttributes(StreamingSearchCluster sc, DeployLogger logger) { - for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { + private static void warnStreamingAttributes(String cluster, Schema schema, DeployLogger logger) { + for (ImmutableSDField sd : schema.allConcreteFields()) { if (sd.doesAttributing()) { - warnStreamingAttribute(sc, sd, logger); + warnStreamingAttribute(cluster, schema.getName(), sd, logger); } } } - private static void warnStreamingAttribute(StreamingSearchCluster sc, ImmutableSDField sd, DeployLogger logger) { + private static void warnStreamingAttribute(String cluster, String schema, ImmutableSDField sd, DeployLogger logger) { // If the field is numeric, we can't print this, because we may have converted the field to // attribute indexing ourselves (IntegerIndex2Attribute) if (sd.getDataType() instanceof NumericDataType) return; @@ -68,25 +68,25 @@ public class StreamingValidator implements Validator { for (var fieldAttribute : sd.getAttributes().values()) { if (fieldAttribute.hnswIndexParams().isPresent()) { logger.logApplicationPackage(Level.WARNING, - "For streaming search cluster '" + sc.getClusterName() + + "For search cluster '" + cluster + "', schema '" + schema + "', SD field '" + sd.getName() + "': hnsw index is not relevant and not supported, ignoring setting"); } } return; } - logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + + logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster + "', SD field '" + sd.getName() + "': 'attribute' has same match semantics as 'index'."); } - private static void failStreamingDocumentReferences(Context context, StreamingSearchCluster sc) { - for (Attribute attribute : sc.derived().getAttributeFields().attributes()) { + private static void failStreamingDocumentReferences(String cluster, DerivedConfiguration derived, Context context) { + for (Attribute attribute : derived.getAttributeFields().attributes()) { DataType dataType = attribute.getDataType(); if (dataType instanceof NewDocumentReferenceDataType) { - String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " + + String errorMessage = String.format("For search cluster '%s', schema '%s': Attribute '%s' has type '%s'. " + "Document references and imported fields are not allowed in streaming search.", - sc.getClusterName(), attribute.getName(), dataType.getName()); + cluster, derived.getSchema().getName(), attribute.getName(), dataType.getName()); context.illegal(errorMessage); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 3b89467299d..0d42219dade 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; -import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; @@ -12,11 +11,11 @@ import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder; import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.search.StreamingSearchCluster; +import com.yahoo.vespa.model.search.DocumentDatabase; +import com.yahoo.vespa.model.search.SearchCluster; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -31,39 +30,47 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> { ContentCluster nextCluster = context.model().getContentClusters().get(clusterName); if (nextCluster != null) { - List<StreamingSearchCluster> nextStreamingClusters = nextCluster.getSearch().getStreamingClusters(); - currentCluster.getSearch().getStreamingClusters().forEach(currentStreamingCluster -> { - Optional<StreamingSearchCluster> nextStreamingCluster = findStreamingCluster(currentStreamingCluster.getClusterName(), nextStreamingClusters); - nextStreamingCluster.ifPresent(streamingSearchCluster -> validateStreamingCluster(currentCluster, currentStreamingCluster, - nextCluster, streamingSearchCluster).forEach(context::require)); + var nextStreamingClusters = nextCluster.getSearch().getClusters(); + currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> { + SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName()); + if (nextStreamingCluster != null) { + validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require); + } }); } }); } - private static Optional<StreamingSearchCluster> findStreamingCluster(String clusterName, List<StreamingSearchCluster> clusters) { - return clusters.stream() - .filter(cluster -> cluster.getClusterName().equals(clusterName)) - .findFirst(); + private static List<VespaConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster, + SearchCluster currentStreamingCluster, + ContentCluster nextCluster, + SearchCluster nextStreamingCluster) { + List<VespaConfigChangeAction> result = new ArrayList<>(); + + for (DocumentDatabase currentDB : currentStreamingCluster.getDocumentDbs()) { + DocumentDatabase nextDB = nextStreamingCluster.getDocumentDB(currentDB.getName()); + if (nextDB != null) { + result.addAll(validateDocumentDB(currentCluster, currentDB, nextCluster, nextDB)); + } + } + return result; } - private static List<ConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster, - StreamingSearchCluster currentStreamingCluster, - ContentCluster nextCluster, - StreamingSearchCluster nextStreamingCluster) { + private static List<VespaConfigChangeAction> validateDocumentDB(ContentCluster currentCluster, DocumentDatabase currentDB, + ContentCluster nextCluster, DocumentDatabase nextDB) { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(validateDocumentTypeChanges(currentCluster.id(), - getDocumentType(currentCluster, currentStreamingCluster), - getDocumentType(nextCluster, nextStreamingCluster))); + getDocumentType(currentCluster, currentDB), + getDocumentType(nextCluster, nextDB))); result.addAll(validateAttributeFastAccessAdded(currentCluster.id(), - currentStreamingCluster.derived().getAttributeFields(), - nextStreamingCluster.derived().getAttributeFields())); + currentDB.getDerivedConfiguration().getAttributeFields(), + nextDB.getDerivedConfiguration().getAttributeFields())); result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(), - currentStreamingCluster.derived().getAttributeFields(), - nextStreamingCluster.derived().getAttributeFields())); + currentDB.getDerivedConfiguration().getAttributeFields(), + nextDB.getDerivedConfiguration().getAttributeFields())); - return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName()); + return modifyActions(result, getSearchNodeServices(nextCluster), nextDB.getName()); } private static List<VespaConfigChangeAction> validateDocumentTypeChanges(ClusterSpec.Id id, @@ -72,8 +79,8 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(); } - private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) { - return cluster.getDocumentDefinitions().get(streamingCluster.getDocTypeName()); + private static NewDocumentType getDocumentType(ContentCluster cluster, DocumentDatabase db) { + return cluster.getDocumentDefinitions().get(db.getName()); } private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(ClusterSpec.Id id, @@ -110,7 +117,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { .toList(); } - private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, + private static List<VespaConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, List<ServiceInfo> services, String docTypeName) { return result.stream() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 125d3b10512..9d2a7e722ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.Objects; import java.util.TreeMap; import java.util.function.Predicate; @@ -104,12 +103,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog"); var search = new ContentSearchCluster(ancestor, clusterName, - deployState.getProperties().featureFlags(), - documentDefinitions, - globallyDistributedDocuments, - getFlushOnShutdown(flushOnShutdownElem), - syncTransactionLog, - fractionOfMemoryReserved); + deployState.getProperties().featureFlags(), + documentDefinitions, + globallyDistributedDocuments, + getFlushOnShutdown(flushOnShutdownElem), + syncTransactionLog, + fractionOfMemoryReserved); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { @@ -117,8 +116,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } search.setResourceLimits(resourceLimits); - buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); - buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); + buildSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -130,73 +128,18 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { - var schemas = deployState.getSchemas(); - for (var candidate : schemas) { - if (candidate.getName().equals(docTypeName)) { - return candidate; - } - } - return null; - } - - private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { + private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) return; - if (docElem == null) { - return; + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); } - for (ModelElement docType : docElem.subElements("document")) { - String docTypeName = docType.stringAttribute("type"); - String mode = docType.stringAttribute("mode"); - var schema = findResponsibleSchema(deployState, docTypeName); - if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { - buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); - } - } - } - - private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, - ContentSearchCluster search, ModelElement docType) { - String docTypeName = docType.stringAttribute("type"); - StreamingSearchCluster cluster = new StreamingSearchCluster(search, - clusterName + "." + docTypeName, - 0, - docTypeName, - clusterName); - search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); - } - - private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { - List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); - if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); - } - - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); - } - } - - private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { - List<ModelElement> indexedDefs = new ArrayList<>(); - ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) { - return indexedDefs; - } - - for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.stringAttribute("mode"); - if ("index".equals(mode)) { - indexedDefs.add(docType); - } - } - return indexedDefs; + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); } } @@ -247,9 +190,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - addCluster(cluster); + if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { + addCluster(cluster); + } } + private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -257,8 +203,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); + if (schema.isDocumentsOnly()) continue; - sc.add(new SchemaInfo(schema, deployState.rankProfileRegistry(), null)); + sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); } } @@ -282,14 +229,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> * with indexing, null if it has both or none. */ public Boolean isStreaming() { - boolean hasStreaming = false; - boolean hasIndexed = false; - for (var cluster : clusters.values()) { - if (cluster.isStreaming()) - hasStreaming = true; - else - hasIndexed = true; - } + if (indexedCluster == null) return false; + boolean hasStreaming = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING); + boolean hasIndexed = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.INDEX); if (hasIndexed == hasStreaming) return null; return hasStreaming; } @@ -356,14 +298,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> this.redundancy = redundancy; } - private Optional<StreamingSearchCluster> findStreamingCluster(String docType) { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.schemas().get(docType) != null) - .findFirst(); - } - public List<StreamingSearchCluster> getStreamingClusters() { return getClusters().values().stream() .filter(StreamingSearchCluster.class::isInstance) @@ -382,13 +316,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } private boolean hasIndexingModeStreaming(NewDocumentType type) { - return findStreamingCluster(type.getFullName().getName()).isPresent(); + if (indexedCluster == null) return false; + return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING; } private boolean hasIndexingModeIndexed(NewDocumentType type) { - return !hasIndexingModeStreaming(type) - && hasIndexedCluster() - && getIndexed().hasDocumentDB(type.getFullName().getName()); + if (indexedCluster == null) return false; + return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX; } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -397,7 +331,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedCluster = false; + boolean hasAnyNonIndexedSchema = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -409,13 +343,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedCluster = true; - findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedSchema = true; + indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedCluster = true; + hasAnyNonIndexedSchema = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -424,7 +358,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> builder.documentdb(ddbB); } - if (hasAnyNonIndexedCluster) { + if (hasAnyNonIndexedSchema) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index b51185ddac2..4a37b27d1c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -65,7 +65,7 @@ public class IndexedSearchCluster extends SearchCluster implements for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; var db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(spec.fullSchema(), deployState, false)); + new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode())); add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index f19af8b62ca..990a8bd25aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -8,14 +8,23 @@ import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -35,7 +44,36 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer private Double queryTimeout; private Double visibilityDelay = 0.0; private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>(); - private final List<DocumentDatabase> documentDbs = new LinkedList<>(); + private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); + private final List<LegacyStreamingproxy> legacyproxy = new ArrayList<>(); + + private static class LegacyStreamingproxy extends TreeConfigProducer<AnyConfigProducer> implements + AttributesConfig.Producer, + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer, + JuniperrcConfig.Producer, + SummaryConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer + { + private final DocumentDatabase db; + LegacyStreamingproxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, DocumentDatabase db) { + super(parent, "cluster." + clusterName + "." + db.getName()); + this.db = new DocumentDatabase(this, db.getName(), db.getDerivedConfiguration()); + } + @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); } + + } public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -43,29 +81,28 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer this.index = index; } + public String getStorageRouteSpec() { return getClusterName(); } + public void add(SchemaInfo schema) { schemas.put(schema.name(), schema); } public void add(DocumentDatabase db) { - documentDbs.add(db); + if (db.getDerivedConfiguration().isStreaming()) { + legacyproxy.add(new LegacyStreamingproxy((TreeConfigProducer<AnyConfigProducer>) getParent(), clusterName, db)); + } + documentDbs.put(db.getName(), db); } public boolean hasDocumentDB(String name) { - for (DocumentDatabase db : documentDbs) { - if (db.getName().equals(name)) { - return true; - } - } - return false; + return documentDbs.containsKey(name); + } + public DocumentDatabase getDocumentDB(String name) { + return documentDbs.get(name); } public String getConfigId(String name) { - for (DocumentDatabase db : documentDbs) { - if (db.getName().equals(name)) { - return db.getConfigId(); - } - } - return ""; + DocumentDatabase db = documentDbs.get(name); + return db != null ? db.getConfigId() : ""; } /** Returns the schemas that should be active in this cluster. Note: These are added during processing. */ @@ -80,7 +117,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer /** Returns the document databases contained in this cluster */ public List<DocumentDatabase> getDocumentDbs() { - return Collections.unmodifiableList(documentDbs); + return documentDbs.values().stream().toList(); } public String getClusterName() { return clusterName; } @@ -97,11 +134,9 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public final int getClusterIndex() { return index; } public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) { - for (DocumentDatabase sdoc : documentDbs) { - if (sdoc.getName().equals(documentType)) { - fillDocumentDBConfig(sdoc, builder); - return; - } + DocumentDatabase db = documentDbs.get(documentType); + if (db != null) { + fillDocumentDBConfig(db, builder); } } @@ -112,7 +147,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { - for (DocumentDatabase db : documentDbs) { + for (DocumentDatabase db : documentDbs.values()) { var docDb = new DocumentdbInfoConfig.Documentdb.Builder() .name(db.getName()) .mode(db.getDerivedConfiguration().isStreaming() @@ -123,21 +158,21 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer } @Override public void getConfig(IndexInfoConfig.Builder builder) { - new Join(documentDbs).getConfig(builder); + new Join(documentDbs.values()).getConfig(builder); } @Override public void getConfig(SchemaInfoConfig.Builder builder) { - new Join(documentDbs).getConfig(builder); + new Join(documentDbs.values()).getConfig(builder); } @Override public void getConfig(IlscriptsConfig.Builder builder) { - new Join(documentDbs).getConfig(builder); + new Join(documentDbs.values()).getConfig(builder); } public void getConfig(AttributesConfig.Builder builder) { - new Join(documentDbs).getConfig(builder); + new Join(documentDbs.values()).getConfig(builder); } @Override @@ -167,7 +202,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer * that is handled (by delegating to this) by the {@link IndexedSearchCluster} * which is the parent to this. This avoids building the config multiple times. */ - private record Join(List<DocumentDatabase> docDbs) { + private record Join(Collection<DocumentDatabase> docDbs) { public void getConfig(IndexInfoConfig.Builder builder) { for (DocumentDatabase docDb : docDbs) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 3f15bc90b8f..e188a086614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.AttributeFields; import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; @@ -64,7 +65,7 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(schema, deployState, true))); + add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING))); } protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { |