diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-03-13 22:31:36 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-13 22:31:36 +0100 |
commit | 14ebb1d9b9e35ac7b853f0fe919826899dd08d2f (patch) | |
tree | c00562090b5bb8bb55da461e93f085d6910ad5a6 /config-model | |
parent | 7f94821395235f2bb75de793fb077622327c359d (diff) |
Revert "Balder/single searchcluster"
Diffstat (limited to 'config-model')
22 files changed, 374 insertions, 234 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index a7f66966538..b35918b3460 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -28,7 +28,7 @@ import java.io.Writer; public class DerivedConfiguration { private final Schema schema; - private final SchemaInfo.IndexMode indexMode; + private final boolean isStreaming; private Summaries summaries; private Juniperrc juniperrc; private AttributeFields attributeFields; @@ -57,8 +57,7 @@ public class DerivedConfiguration { } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), - schema, SchemaInfo.IndexMode.INDEX); + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false); } /** @@ -68,8 +67,8 @@ public class DerivedConfiguration { * argument is live. Which means that this object will be inconsistent if the given * schema is later modified. */ - public DerivedConfiguration(DeployState deployState, Schema schema, SchemaInfo.IndexMode indexMode) { - this.indexMode = indexMode; + public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { + this.isStreaming = isStreaming; try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -82,9 +81,9 @@ public class DerivedConfiguration { summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); - indexingScript = new IndexingScript(schema, isStreaming()); - indexInfo = new IndexInfo(schema, isStreaming()); - schemaInfo = new SchemaInfo(schema, indexMode, deployState.rankProfileRegistry(), summaries); + indexingScript = new IndexingScript(schema, isStreaming); + indexInfo = new IndexInfo(schema, isStreaming); + schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } @@ -154,7 +153,7 @@ public class DerivedConfiguration { } public boolean isStreaming() { - return indexMode == SchemaInfo.IndexMode.STREAMING; + return isStreaming; } public Summaries getSummaries() { @@ -166,11 +165,7 @@ public class DerivedConfiguration { } public void getConfig(AttributesConfig.Builder builder) { - if (isStreaming()) { - getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } else { - getConfig(builder, AttributeFields.FieldSet.ALL); - } + getConfig(builder, AttributeFields.FieldSet.ALL); } public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index f996b2624db..19a045ac444 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -42,32 +42,13 @@ public final class SchemaInfo extends Derived { private final Map<String, RankProfileInfo> rankProfiles; private final Summaries summaries; - private final IndexMode indexMode; - public enum IndexMode {INDEX, STREAMING, STORE_ONLY} - - public SchemaInfo(Schema schema, String indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) { - this(schema, indexMode(indexMode), rankProfileRegistry, summaries); - } - public SchemaInfo(Schema schema, IndexMode indexMode, RankProfileRegistry rankProfileRegistry, Summaries summaries) { + public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, Summaries summaries) { this.schema = schema; this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); this.summaries = summaries; - this.indexMode = indexMode; } - private static IndexMode indexMode(String mode) { - if (mode == null) return IndexMode.INDEX; - return switch (mode) { - case "index" -> IndexMode.INDEX; - case "streaming" -> IndexMode.STREAMING; - case "store-only" -> IndexMode.STORE_ONLY; - default -> IndexMode.STORE_ONLY; - }; - } - - public IndexMode getIndexMode() { return indexMode; } - public String name() { return schema.getName(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java index 7b7c92a43e0..12f3f025996 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java @@ -28,10 +28,10 @@ public class ComplexFieldsWithStructFieldAttributesValidator implements Validato public void validate(Context context) { List<SearchCluster> searchClusters = context.model().getSearchClusters(); for (SearchCluster cluster : searchClusters) { + if (cluster.isStreaming()) continue; + for (SchemaInfo spec : cluster.schemas().values()) { - if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) { - validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger()); - } + validateComplexFields(context, cluster.getClusterName(), spec.fullSchema(), context.deployState().getDeployLogger()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java index 2d553c84f48..3dbba081400 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.schema.Schema; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.model.application.validation.Validation.Context; @@ -25,10 +24,11 @@ public class ComplexFieldsWithStructFieldIndexesValidator implements Validator { @Override public void validate(Context context) { for (var cluster : context.model().getSearchClusters()) { + if (cluster.isStreaming()) { + continue; + } for (var spec : cluster.schemas().values()) { - if (spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) { - validateComplexFields(context, cluster.getClusterName(), spec.fullSchema()); - } + validateComplexFields(context, cluster.getClusterName(), spec.fullSchema()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index e1c64422653..15d293e4abc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -23,9 +23,10 @@ public class NoPrefixForIndexes implements Validator { @Override public void validate(Context context) { for (SearchCluster cluster : context.model().getSearchClusters()) { - for (DocumentDatabase docDb : cluster.getDocumentDbs()) { - DerivedConfiguration sdConfig = docDb.getDerivedConfiguration(); - if ( ! sdConfig.isStreaming() ) { + if (cluster instanceof IndexedSearchCluster) { + IndexedSearchCluster sc = (IndexedSearchCluster) cluster; + for (DocumentDatabase docDb : sc.getDocumentDbs()) { + DerivedConfiguration sdConfig = docDb.getDerivedConfiguration(); Schema schema = sdConfig.getSchema(); for (ImmutableSDField field : schema.allConcreteFields()) { if (field.doesIndexing()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 736665b2184..03076b64d0c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -69,7 +69,7 @@ public class RankSetupValidator implements Validator { String schemaDir = clusterDir + schemaName + "/"; writeConfigs(schemaDir, docDb); writeExtraVerifyRankSetupConfig(schemaDir, docDb); - if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, docDb.getDerivedConfiguration().isStreaming())) { + if (!validate(context, "dir:" + schemaDir, sc, schemaName, cfgDir, sc.isStreaming())) { return; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index cd8094f8c12..6e21adc4fe4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -31,11 +31,15 @@ public class SearchDataTypeValidator implements Validator { public void validate(Context context) { List<SearchCluster> clusters = context.model().getSearchClusters(); for (SearchCluster cluster : clusters) { + if (cluster.isStreaming()) { + continue; + } for (SchemaInfo spec : cluster.schemas().values()) { SDDocumentType docType = spec.fullSchema().getDocument(); - if (docType != null && spec.getIndexMode() == SchemaInfo.IndexMode.INDEX) { - validateDocument(context, cluster, spec.fullSchema(), docType); + if (docType == null) { + continue; } + validateDocument(context, cluster, spec.fullSchema(), docType); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 0e9d3bf7016..c0ad55fc8f4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -6,14 +6,12 @@ import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.TensorDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.schema.Schema; -import com.yahoo.schema.derived.DerivedConfiguration; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.MatchType; import com.yahoo.vespa.model.application.validation.Validation.Context; import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import java.util.List; import java.util.logging.Level; @@ -27,21 +25,20 @@ public class StreamingValidator implements Validator { public void validate(Context context) { List<SearchCluster> searchClusters = context.model().getSearchClusters(); for (SearchCluster cluster : searchClusters) { - for (SchemaInfo schemaInfo : cluster.schemas().values()) { - if (schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STREAMING) { - var deployLogger = context.deployState().getDeployLogger(); - warnStreamingAttributes(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger); - warnStreamingGramMatching(cluster.getClusterName(), schemaInfo.fullSchema(), deployLogger); - failStreamingDocumentReferences(cluster.getClusterName(), cluster.getDocumentDB(schemaInfo.name()).getDerivedConfiguration(), context); - } - } + if ( ! cluster.isStreaming()) continue; + + var streamingCluster = (StreamingSearchCluster)cluster; + warnStreamingAttributes(streamingCluster, context.deployState().getDeployLogger()); + warnStreamingGramMatching(streamingCluster, context.deployState().getDeployLogger()); + failStreamingDocumentReferences(context, streamingCluster); } } - private static void warnStreamingGramMatching(String cluster, Schema schema, DeployLogger logger) { - for (ImmutableSDField sd : schema.allConcreteFields()) { + private static void warnStreamingGramMatching(StreamingSearchCluster sc, DeployLogger logger) { + for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { if (sd.getMatching().getType() == MatchType.GRAM) { - logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster + "', schema '" + schema.getName() + + logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + + sc.getClusterName() + "', SD field '" + sd.getName() + "': n-gram matching is not supported for streaming search."); } @@ -50,16 +47,19 @@ public class StreamingValidator implements Validator { /** * Warn if one or more attributes are defined in a streaming search cluster SD. + * + * @param sc a search cluster to be checked for attributes in streaming search + * @param logger a DeployLogger */ - private static void warnStreamingAttributes(String cluster, Schema schema, DeployLogger logger) { - for (ImmutableSDField sd : schema.allConcreteFields()) { + private static void warnStreamingAttributes(StreamingSearchCluster sc, DeployLogger logger) { + for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { if (sd.doesAttributing()) { - warnStreamingAttribute(cluster, schema.getName(), sd, logger); + warnStreamingAttribute(sc, sd, logger); } } } - private static void warnStreamingAttribute(String cluster, String schema, ImmutableSDField sd, DeployLogger logger) { + private static void warnStreamingAttribute(StreamingSearchCluster sc, ImmutableSDField sd, DeployLogger logger) { // If the field is numeric, we can't print this, because we may have converted the field to // attribute indexing ourselves (IntegerIndex2Attribute) if (sd.getDataType() instanceof NumericDataType) return; @@ -68,25 +68,25 @@ public class StreamingValidator implements Validator { for (var fieldAttribute : sd.getAttributes().values()) { if (fieldAttribute.hnswIndexParams().isPresent()) { logger.logApplicationPackage(Level.WARNING, - "For search cluster '" + cluster + "', schema '" + schema + + "For streaming search cluster '" + sc.getClusterName() + "', SD field '" + sd.getName() + "': hnsw index is not relevant and not supported, ignoring setting"); } } return; } - logger.logApplicationPackage(Level.WARNING, "For search cluster '" + cluster + + logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + "', SD field '" + sd.getName() + "': 'attribute' has same match semantics as 'index'."); } - private static void failStreamingDocumentReferences(String cluster, DerivedConfiguration derived, Context context) { - for (Attribute attribute : derived.getAttributeFields().attributes()) { + private static void failStreamingDocumentReferences(Context context, StreamingSearchCluster sc) { + for (Attribute attribute : sc.derived().getAttributeFields().attributes()) { DataType dataType = attribute.getDataType(); if (dataType instanceof NewDocumentReferenceDataType) { - String errorMessage = String.format("For search cluster '%s', schema '%s': Attribute '%s' has type '%s'. " + + String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " + "Document references and imported fields are not allowed in streaming search.", - cluster, derived.getSchema().getName(), attribute.getName(), dataType.getName()); + sc.getClusterName(), attribute.getName(), dataType.getName()); context.illegal(errorMessage); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 0d42219dade..3b89467299d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; @@ -11,11 +12,11 @@ import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder; import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.search.DocumentDatabase; -import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -30,47 +31,39 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> { ContentCluster nextCluster = context.model().getContentClusters().get(clusterName); if (nextCluster != null) { - var nextStreamingClusters = nextCluster.getSearch().getClusters(); - currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> { - SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName()); - if (nextStreamingCluster != null) { - validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require); - } + List<StreamingSearchCluster> nextStreamingClusters = nextCluster.getSearch().getStreamingClusters(); + currentCluster.getSearch().getStreamingClusters().forEach(currentStreamingCluster -> { + Optional<StreamingSearchCluster> nextStreamingCluster = findStreamingCluster(currentStreamingCluster.getClusterName(), nextStreamingClusters); + nextStreamingCluster.ifPresent(streamingSearchCluster -> validateStreamingCluster(currentCluster, currentStreamingCluster, + nextCluster, streamingSearchCluster).forEach(context::require)); }); } }); } - private static List<VespaConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster, - SearchCluster currentStreamingCluster, - ContentCluster nextCluster, - SearchCluster nextStreamingCluster) { - List<VespaConfigChangeAction> result = new ArrayList<>(); - - for (DocumentDatabase currentDB : currentStreamingCluster.getDocumentDbs()) { - DocumentDatabase nextDB = nextStreamingCluster.getDocumentDB(currentDB.getName()); - if (nextDB != null) { - result.addAll(validateDocumentDB(currentCluster, currentDB, nextCluster, nextDB)); - } - } - return result; + private static Optional<StreamingSearchCluster> findStreamingCluster(String clusterName, List<StreamingSearchCluster> clusters) { + return clusters.stream() + .filter(cluster -> cluster.getClusterName().equals(clusterName)) + .findFirst(); } - private static List<VespaConfigChangeAction> validateDocumentDB(ContentCluster currentCluster, DocumentDatabase currentDB, - ContentCluster nextCluster, DocumentDatabase nextDB) { + private static List<ConfigChangeAction> validateStreamingCluster(ContentCluster currentCluster, + StreamingSearchCluster currentStreamingCluster, + ContentCluster nextCluster, + StreamingSearchCluster nextStreamingCluster) { List<VespaConfigChangeAction> result = new ArrayList<>(); result.addAll(validateDocumentTypeChanges(currentCluster.id(), - getDocumentType(currentCluster, currentDB), - getDocumentType(nextCluster, nextDB))); + getDocumentType(currentCluster, currentStreamingCluster), + getDocumentType(nextCluster, nextStreamingCluster))); result.addAll(validateAttributeFastAccessAdded(currentCluster.id(), - currentDB.getDerivedConfiguration().getAttributeFields(), - nextDB.getDerivedConfiguration().getAttributeFields())); + currentStreamingCluster.derived().getAttributeFields(), + nextStreamingCluster.derived().getAttributeFields())); result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(), - currentDB.getDerivedConfiguration().getAttributeFields(), - nextDB.getDerivedConfiguration().getAttributeFields())); + currentStreamingCluster.derived().getAttributeFields(), + nextStreamingCluster.derived().getAttributeFields())); - return modifyActions(result, getSearchNodeServices(nextCluster), nextDB.getName()); + return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName()); } private static List<VespaConfigChangeAction> validateDocumentTypeChanges(ClusterSpec.Id id, @@ -79,8 +72,8 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { return new DocumentTypeChangeValidator(id, currentDocType, nextDocType).validate(); } - private static NewDocumentType getDocumentType(ContentCluster cluster, DocumentDatabase db) { - return cluster.getDocumentDefinitions().get(db.getName()); + private static NewDocumentType getDocumentType(ContentCluster cluster, StreamingSearchCluster streamingCluster) { + return cluster.getDocumentDefinitions().get(streamingCluster.getDocTypeName()); } private static List<VespaConfigChangeAction> validateAttributeFastAccessAdded(ClusterSpec.Id id, @@ -117,7 +110,7 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { .toList(); } - private static List<VespaConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, + private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, List<ServiceInfo> services, String docTypeName) { return result.stream() diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index c98e2dac1d9..035ae4a06f5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import java.util.Collection; import java.util.HashSet; @@ -178,7 +179,10 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> } scB.rankprofiles_configid(sys.getConfigId()); scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); - scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(sys.getStorageRouteSpec())); + if ( ! (sys instanceof IndexedSearchCluster)) { + scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). + routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); + } builder.searchcluster(scB); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index e2fe660cbe9..125d3b10512 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; +import java.util.Optional; import java.util.Objects; import java.util.TreeMap; import java.util.function.Predicate; @@ -102,12 +104,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog"); var search = new ContentSearchCluster(ancestor, clusterName, - deployState.getProperties().featureFlags(), - documentDefinitions, - globallyDistributedDocuments, - getFlushOnShutdown(flushOnShutdownElem), - syncTransactionLog, - fractionOfMemoryReserved); + deployState.getProperties().featureFlags(), + documentDefinitions, + globallyDistributedDocuments, + getFlushOnShutdown(flushOnShutdownElem), + syncTransactionLog, + fractionOfMemoryReserved); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { @@ -115,7 +117,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } search.setResourceLimits(resourceLimits); - buildSearchCluster(deployState, clusterElem, clusterName, search); + buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); + buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -127,18 +130,73 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { + private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { + var schemas = deployState.getSchemas(); + for (var candidate : schemas) { + if (candidate.getName().equals(docTypeName)) { + return candidate; + } + } + return null; + } + + private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) return; - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); + if (docElem == null) { + return; } - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); + for (ModelElement docType : docElem.subElements("document")) { + String docTypeName = docType.stringAttribute("type"); + String mode = docType.stringAttribute("mode"); + var schema = findResponsibleSchema(deployState, docTypeName); + if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { + buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); + } + } + } + + private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, + ContentSearchCluster search, ModelElement docType) { + String docTypeName = docType.stringAttribute("type"); + StreamingSearchCluster cluster = new StreamingSearchCluster(search, + clusterName + "." + docTypeName, + 0, + docTypeName, + clusterName); + search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); + } + + private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { + List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); + if (!indexedDefs.isEmpty()) { + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); + } + + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); + } + } + + private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { + List<ModelElement> indexedDefs = new ArrayList<>(); + ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) { + return indexedDefs; + } + + for (ModelElement docType : docElem.subElements("document")) { + String mode = docType.stringAttribute("mode"); + if ("index".equals(mode)) { + indexedDefs.add(docType); + } + } + return indexedDefs; } } @@ -189,12 +247,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { - addCluster(cluster); - } + addCluster(cluster); } - private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -202,9 +257,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); - if (schema.isDocumentsOnly()) continue; - sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); + sc.add(new SchemaInfo(schema, deployState.rankProfileRegistry(), null)); } } @@ -228,9 +282,14 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> * with indexing, null if it has both or none. */ public Boolean isStreaming() { - if (indexedCluster == null) return false; - boolean hasStreaming = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING); - boolean hasIndexed = indexedCluster.schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.INDEX); + boolean hasStreaming = false; + boolean hasIndexed = false; + for (var cluster : clusters.values()) { + if (cluster.isStreaming()) + hasStreaming = true; + else + hasIndexed = true; + } if (hasIndexed == hasStreaming) return null; return hasStreaming; } @@ -297,6 +356,21 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> this.redundancy = redundancy; } + private Optional<StreamingSearchCluster> findStreamingCluster(String docType) { + return getClusters().values().stream() + .filter(StreamingSearchCluster.class::isInstance) + .map(StreamingSearchCluster.class::cast) + .filter(ssc -> ssc.schemas().get(docType) != null) + .findFirst(); + } + + public List<StreamingSearchCluster> getStreamingClusters() { + return getClusters().values().stream() + .filter(StreamingSearchCluster.class::isInstance) + .map(StreamingSearchCluster.class::cast) + .toList(); + } + public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); } public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); } public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); } @@ -308,13 +382,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } private boolean hasIndexingModeStreaming(NewDocumentType type) { - if (indexedCluster == null) return false; - return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING; + return findStreamingCluster(type.getFullName().getName()).isPresent(); } private boolean hasIndexingModeIndexed(NewDocumentType type) { - if (indexedCluster == null) return false; - return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX; + return !hasIndexingModeStreaming(type) + && hasIndexedCluster() + && getIndexed().hasDocumentDB(type.getFullName().getName()); } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -323,7 +397,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedSchema = false; + boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -335,13 +409,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedSchema = true; - indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedCluster = true; + findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); + getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedSchema = true; + hasAnyNonIndexedCluster = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -350,7 +424,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> builder.documentdb(ddbB); } - if (hasAnyNonIndexedSchema) { + if (hasAnyNonIndexedCluster) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4a37b27d1c7..b51185ddac2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -65,7 +65,7 @@ public class IndexedSearchCluster extends SearchCluster implements for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; var db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode())); + new DerivedConfiguration(spec.fullSchema(), deployState, false)); add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 990a8bd25aa..f19af8b62ca 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -8,23 +8,14 @@ import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.SummaryConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.config.search.summary.JuniperrcConfig; -import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; -import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -44,36 +35,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer private Double queryTimeout; private Double visibilityDelay = 0.0; private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>(); - private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); - private final List<LegacyStreamingproxy> legacyproxy = new ArrayList<>(); - - private static class LegacyStreamingproxy extends TreeConfigProducer<AnyConfigProducer> implements - AttributesConfig.Producer, - RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - RankingExpressionsConfig.Producer, - OnnxModelsConfig.Producer, - JuniperrcConfig.Producer, - SummaryConfig.Producer, - VsmsummaryConfig.Producer, - VsmfieldsConfig.Producer - { - private final DocumentDatabase db; - LegacyStreamingproxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, DocumentDatabase db) { - super(parent, "cluster." + clusterName + "." + db.getName()); - this.db = new DocumentDatabase(this, db.getName(), db.getDerivedConfiguration()); - } - @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); } - - } + private final List<DocumentDatabase> documentDbs = new LinkedList<>(); public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -81,28 +43,29 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer this.index = index; } - public String getStorageRouteSpec() { return getClusterName(); } - public void add(SchemaInfo schema) { schemas.put(schema.name(), schema); } public void add(DocumentDatabase db) { - if (db.getDerivedConfiguration().isStreaming()) { - legacyproxy.add(new LegacyStreamingproxy((TreeConfigProducer<AnyConfigProducer>) getParent(), clusterName, db)); - } - documentDbs.put(db.getName(), db); + documentDbs.add(db); } public boolean hasDocumentDB(String name) { - return documentDbs.containsKey(name); - } - public DocumentDatabase getDocumentDB(String name) { - return documentDbs.get(name); + for (DocumentDatabase db : documentDbs) { + if (db.getName().equals(name)) { + return true; + } + } + return false; } public String getConfigId(String name) { - DocumentDatabase db = documentDbs.get(name); - return db != null ? db.getConfigId() : ""; + for (DocumentDatabase db : documentDbs) { + if (db.getName().equals(name)) { + return db.getConfigId(); + } + } + return ""; } /** Returns the schemas that should be active in this cluster. Note: These are added during processing. */ @@ -117,7 +80,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer /** Returns the document databases contained in this cluster */ public List<DocumentDatabase> getDocumentDbs() { - return documentDbs.values().stream().toList(); + return Collections.unmodifiableList(documentDbs); } public String getClusterName() { return clusterName; } @@ -134,9 +97,11 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public final int getClusterIndex() { return index; } public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) { - DocumentDatabase db = documentDbs.get(documentType); - if (db != null) { - fillDocumentDBConfig(db, builder); + for (DocumentDatabase sdoc : documentDbs) { + if (sdoc.getName().equals(documentType)) { + fillDocumentDBConfig(sdoc, builder); + return; + } } } @@ -147,7 +112,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { - for (DocumentDatabase db : documentDbs.values()) { + for (DocumentDatabase db : documentDbs) { var docDb = new DocumentdbInfoConfig.Documentdb.Builder() .name(db.getName()) .mode(db.getDerivedConfiguration().isStreaming() @@ -158,21 +123,21 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer } @Override public void getConfig(IndexInfoConfig.Builder builder) { - new Join(documentDbs.values()).getConfig(builder); + new Join(documentDbs).getConfig(builder); } @Override public void getConfig(SchemaInfoConfig.Builder builder) { - new Join(documentDbs.values()).getConfig(builder); + new Join(documentDbs).getConfig(builder); } @Override public void getConfig(IlscriptsConfig.Builder builder) { - new Join(documentDbs.values()).getConfig(builder); + new Join(documentDbs).getConfig(builder); } public void getConfig(AttributesConfig.Builder builder) { - new Join(documentDbs.values()).getConfig(builder); + new Join(documentDbs).getConfig(builder); } @Override @@ -202,7 +167,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer * that is handled (by delegating to this) by the {@link IndexedSearchCluster} * which is the parent to this. This avoids building the config multiple times. */ - private record Join(Collection<DocumentDatabase> docDbs) { + private record Join(List<DocumentDatabase> docDbs) { public void getConfig(IndexInfoConfig.Builder builder) { for (DocumentDatabase docDb : docDbs) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java new file mode 100644 index 00000000000..3f15bc90b8f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -0,0 +1,99 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; + +/** + * A search cluster of type streaming. + * + * @author baldersheim + * @author vegardh + */ +public class StreamingSearchCluster extends SearchCluster implements + AttributesConfig.Producer, + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer, + JuniperrcConfig.Producer, + SummaryConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer +{ + private final String storageRouteSpec; + private final AttributesProducer attributesConfig; + private final String docTypeName; + + public StreamingSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, + String docTypeName, String storageRouteSpec) { + super(parent, clusterName, index); + attributesConfig = new AttributesProducer(parent, docTypeName); + this.docTypeName = docTypeName; + this.storageRouteSpec = storageRouteSpec; + } + + @Override + protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } + public final String getStorageRouteSpec() { return storageRouteSpec; } + + public String getDocTypeName() { return docTypeName; } + + public DerivedConfiguration derived() { return db().getDerivedConfiguration(); } + + @Override + public void deriveFromSchemas(DeployState deployState) { + if (schemas().isEmpty()) return; + if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size()); + + Schema schema = schemas().values().stream().findAny().get().fullSchema(); + if ( ! schema.getName().equals(docTypeName)) + throw new IllegalArgumentException("Document type name '" + docTypeName + + "' must be the same as the schema name '" + schema.getName() + "'"); + add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(schema, deployState, true))); + } + + protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { + super.fillDocumentDBConfig(sdoc, ddbB); + ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up + } + + private DocumentDatabase db() { return getDocumentDbs().get(0); } + + // These are temporary until backend uses correct config id. + @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);} + + private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { + + AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType) { + super(parent, docType); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java index 52ddf238c96..7d7ed2b23eb 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java @@ -47,12 +47,12 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { TestProperties properties, ApplicationBuilder builder, DeployLogger logger) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(new DeployState.Builder() - .properties(properties) - .deployLogger(logger) - .rankProfileRegistry(builder.getRankProfileRegistry()) - .queryProfiles(builder.getQueryProfileRegistry()) - .build(), builder.getSchema(schemaName), SchemaInfo.IndexMode.INDEX); + DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(schemaName), + new DeployState.Builder().properties(properties) + .deployLogger(logger) + .rankProfileRegistry(builder.getRankProfileRegistry()) + .queryProfiles(builder.getQueryProfileRegistry()) + .build(), false); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java index c940afce428..2beab3d5ea9 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java @@ -41,13 +41,12 @@ public class SchemaToDerivedConfigExporter { var props = new TestProperties(); var logger = new TestableDeployLogger(); var builder = ApplicationBuilder.createFromDirectory(dirPath, new MockFileRegistry(), logger, props); - var derived = new DerivedConfiguration( + var derived = new DerivedConfiguration(builder.getSchema(null), new DeployState.Builder().properties(props) .deployLogger(logger) .rankProfileRegistry(builder.getRankProfileRegistry()) .queryProfiles(builder.getQueryProfileRegistry()) - .build(), - builder.getSchema(null), SchemaInfo.IndexMode.INDEX); + .build(), false); exportConfig(dirPath, derived, builder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java index bdac1b1e268..5397c30f2bc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java @@ -29,7 +29,7 @@ public class StreamingValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/document_references_validation/") .create(); }); - assertTrue(exception.getMessage().contains("For search cluster 'content', schema 'ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " + + assertTrue(exception.getMessage().contains("For streaming search cluster 'content.ad': Attribute 'campaign_ref' has type 'Reference<campaign>'. " + "Document references and imported fields are not allowed in streaming search.")); } @@ -52,7 +52,7 @@ public class StreamingValidatorTest { "attribute { distance-metric: euclidean } }"); var warnings = filter(logger.warnings); assertEquals(1, warnings.size()); - assertEquals("For search cluster 'content', schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting", + assertEquals("For streaming search cluster 'content.test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting", warnings.get(0)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java index 28eabd18539..ee64ceb6969 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java @@ -92,9 +92,9 @@ public class StreamingSchemaClusterChangeValidatorTest { private static final String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }"; private static final String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }"; private static final List<ServiceInfo> FOO_SERVICE = - List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null")); + List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null")); private static final List<ServiceInfo> BAR_SERVICE = - List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null")); + List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null")); @Test void changing_field_type_requires_refeed() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 327b169a629..f8adb18a2c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -196,11 +197,12 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = cluster.getSearch(); - assertTrue(s.hasIndexedCluster()); + assertFalse(s.hasIndexedCluster()); assertEquals(1, s.getClusters().size()); - SearchCluster sc = s.getClusters().get(musicClusterId); - assertEquals(musicClusterId, sc.getClusterName()); - assertEquals(musicClusterId, sc.getStorageRouteSpec()); + assertNull(s.getIndexed()); + SearchCluster sc = s.getClusters().get(musicClusterId + ".music"); + assertEquals(musicClusterId + ".music", sc.getClusterName()); + assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec()); assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory); assertEquals(1, cluster.getStorageCluster().getChildren().size()); @@ -235,11 +237,24 @@ public class ContentBuilderTest extends DomBuilderTest { " <node hostalias=\"mockhost\" distribution-key=\"0\"/>" + " </group>" + "</content>"); - ContentSearchCluster s = cluster.getSearch(); - assertTrue(s.hasIndexedCluster()); - assertEquals(2, s.getIndexed().getDocumentDbs().size()); - assertTrue(s.getIndexed().hasDocumentDB("book")); - assertTrue(s.getIndexed().hasDocumentDB("music")); + ContentSearchCluster s; + + s = cluster.getSearch(); + assertFalse(s.hasIndexedCluster()); + assertEquals(2, s.getClusters().size()); + assertNull(s.getIndexed()); + { + String id = musicClusterId + ".book"; + SearchCluster sc = s.getClusters().get(id); + assertEquals(id, sc.getClusterName()); + assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec()); + } + { + String id = musicClusterId + ".music"; + SearchCluster sc = s.getClusters().get(id); + assertEquals(id, sc.getClusterName()); + assertEquals(musicClusterId, ((StreamingSearchCluster) sc).getStorageRouteSpec()); + } assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory); assertEquals(1, cluster.getStorageCluster().getChildren().size()); @@ -348,7 +363,9 @@ public class ContentBuilderTest extends DomBuilderTest { " <node hostalias=\"mockhost\" distribution-key=\"0\" />" + " </group>" + "</content>", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2"))); - ContentSearchCluster s = b.getSearch(); + ContentSearchCluster s; + + s = b.getSearch(); assertTrue(s.hasIndexedCluster()); assertNotNull(s.getIndexed()); assertEquals(1, b.getStorageCluster().getChildren().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index ae08f78c404..6f9127779db 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -163,6 +163,7 @@ public class IndexedTest extends ContentBaseTest { // "transactionlogserver"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); + assertFalse(s.getSearch().hasIndexedCluster()); StorServerConfig.Builder builder = new StorServerConfig.Builder(); s.getStorageCluster().getConfig(builder); @@ -174,6 +175,7 @@ public class IndexedTest extends ContentBaseTest { VespaModel model = getStreamingVespaModel(); ContentCluster s = model.getContentClusters().get("test"); assertNotNull(s); + assertFalse(s.getSearch().hasIndexedCluster()); ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID); assertEquals(1, config.storage().size()); assertEquals("test", config.storage(0).name()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 4cf4068f331..eb4ec1af157 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search.test; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.path.Path; @@ -18,6 +19,7 @@ import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -310,7 +312,7 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); var model = tester.createModelWithMode(mode, sds); - DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); + DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); assertEquals(1, dcfg.documentdb().size()); DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); assertEquals("type", db.name()); @@ -329,11 +331,13 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); var model = tester.createModel(sds, ""); DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); - assertEquals(2, indexed_cfg.documentdb().size()); + assertEquals(1, indexed_cfg.documentdb().size()); var db = indexed_cfg.documentdb(0); assertEquals("a", db.name()); assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); - db = indexed_cfg.documentdb(1); + DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b"); + assertEquals(1, streaming_cfg.documentdb().size()); + db = streaming_cfg.documentdb(0); assertEquals("b", db.name()); assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode()); } @@ -375,7 +379,7 @@ public class DocumentDatabaseTestCase { void testThatAttributesMaxUnCommittedMemoryIsControlledByFeatureFlag() { assertAttributesConfigIndependentOfMode("index", List.of("type1"), List.of("test/search/cluster.test/type1"), - Map.of("type1", List.of("f2", "f2_nfa")), + ImmutableMap.of("type1", List.of("f2", "f2_nfa")), new DeployState.Builder().properties(new TestProperties().maxUnCommittedMemory(193452)), 193452); } @@ -383,20 +387,20 @@ public class DocumentDatabaseTestCase { void testThatAttributesConfigIsProducedForIndexed() { assertAttributesConfigIndependentOfMode("index", List.of("type1"), List.of("test/search/cluster.test/type1"), - Map.of("type1", List.of("f2", "f2_nfa"))); + ImmutableMap.of("type1", List.of("f2", "f2_nfa"))); } @Test void testThatAttributesConfigIsProducedForStreamingForFastAccessFields() { assertAttributesConfigIndependentOfMode("streaming", List.of("type1"), - List.of("test/search/cluster.test/type1"), - Map.of("type1", List.of("f2"))); + List.of("test/search/type1"), + ImmutableMap.of("type1", List.of("f2"))); } @Test void testThatAttributesConfigIsNotProducedForStoreOnlyEvenForFastAccessFields() { assertAttributesConfigIndependentOfMode("store-only", List.of("type1"), - List.of("test/search"), Map.of()); + List.of("test/search"), Collections.emptyMap()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java index 9e4ca630340..388c064daf4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTester.java @@ -2,11 +2,13 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.DataType; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.derived.Summaries; import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; @@ -25,7 +27,7 @@ public class SchemaInfoTester { } public String schemaInfoConfig(Schema schema) { - var schemaInfo = new SchemaInfo(schema, SchemaInfo.IndexMode.INDEX, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties())); + var schemaInfo = new SchemaInfo(schema, new RankProfileRegistry(), new Summaries(schema, new DeployLoggerStub(), new TestProperties())); var schemaInfoConfigBuilder = new SchemaInfoConfig.Builder(); schemaInfo.getConfig(schemaInfoConfigBuilder); var schemaInfoConfig = schemaInfoConfigBuilder.build(); |