From 0e708dc4822ed0d6232b70133381ce0f5fa21092 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 15 Mar 2024 12:00:43 +0100 Subject: Single searchcluster take 4 --- .../yahoo/schema/derived/DerivedConfiguration.java | 6 +- .../StreamingSearchClusterChangeValidator.java | 12 +- .../model/container/search/ContainerSearch.java | 27 +--- .../com/yahoo/vespa/model/content/Content.java | 7 +- .../vespa/model/content/ContentSearchCluster.java | 142 +++++---------------- .../vespa/model/search/IndexedSearchCluster.java | 13 -- .../yahoo/vespa/model/search/SearchCluster.java | 116 +++++++++++++---- .../vespa/model/search/StreamingSearchCluster.java | 100 --------------- .../validation/StreamingValidatorTest.java | 4 +- .../StreamingSchemaClusterChangeValidatorTest.java | 4 +- .../model/builder/xml/dom/ContentBuilderTest.java | 41 ++---- .../search/searchchain/MockSearchClusters.java | 8 +- .../com/yahoo/vespa/model/content/IndexedTest.java | 2 - .../search/test/DocumentDatabaseTestCase.java | 8 +- .../yahoo/search/grouping/GroupingValidator.java | 5 +- .../com/yahoo/search/searchers/QueryValidator.java | 5 +- 16 files changed, 160 insertions(+), 340 deletions(-) delete mode 100644 config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 421f3d5a1d1..a7f66966538 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -166,7 +166,11 @@ public class DerivedConfiguration { } public void getConfig(AttributesConfig.Builder builder) { - getConfig(builder, AttributeFields.FieldSet.ALL); + if (isStreaming()) { + getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } else { + getConfig(builder, AttributeFields.FieldSet.ALL); + } } public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 0d42219dade..39894d95c2c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -30,13 +30,11 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> { ContentCluster nextCluster = context.model().getContentClusters().get(clusterName); if (nextCluster != null) { - var nextStreamingClusters = nextCluster.getSearch().getClusters(); - currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> { - SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName()); - if (nextStreamingCluster != null) { - validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require); - } - }); + if (currentCluster.getSearch().getIndexed() != null && nextCluster.getSearch().getIndexed() != null) { + validateStreamingCluster(currentCluster, currentCluster.getSearch().getIndexed(), + nextCluster, nextCluster.getSearch().getIndexed()) + .forEach(context::require); + } } }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 91622e9fe31..a71d1025ece 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.Dispatcher; @@ -35,15 +34,15 @@ import static com.yahoo.vespa.model.container.PlatformBundles.SEARCH_AND_DOCPROC * @author gjoranv * @author Tony Vaagenes */ -public class ContainerSearch extends ContainerSubsystem - implements +public class ContainerSearch extends ContainerSubsystem implements IndexInfoConfig.Producer, IlscriptsConfig.Producer, QrSearchersConfig.Producer, QueryProfilesConfig.Producer, SemanticRulesConfig.Producer, PageTemplatesConfig.Producer, - SchemaInfoConfig.Producer { + SchemaInfoConfig.Producer +{ public static final String QUERY_PROFILE_REGISTRY_CLASS = CompiledQueryProfileRegistry.class.getName(); @@ -169,25 +168,7 @@ public class ContainerSearch extends ContainerSubsystem @Override public void getConfig(QrSearchersConfig.Builder builder) { - for (int i = 0; i < searchClusters.size(); i++) { - SearchCluster sys = findClusterWithId(searchClusters, i); - var scB = new QrSearchersConfig.Searchcluster.Builder().name(sys.getClusterName()); - for (SchemaInfo spec : sys.schemas().values()) { - scB.searchdef(spec.fullSchema().getName()); - } - scB.rankprofiles_configid(sys.getConfigId()); - scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); - scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(sys.getStorageRouteSpec())); - builder.searchcluster(scB); - } - } - - private static SearchCluster findClusterWithId(List clusters, int index) { - for (SearchCluster sys : clusters) { - if (sys.getClusterIndex() == index) - return sys; - } - throw new IllegalArgumentException("No search cluster with index " + index + " exists"); + searchClusters.forEach(sc -> builder.searchcluster(sc.getQrSearcherConfig())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 1254f8e110a..4ac7a8e442a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -135,8 +135,11 @@ public class Content extends ConfigModel { public static List getSearchClusters(ConfigModelRepo pc) { List clusters = new ArrayList<>(); - for (ContentCluster c : getContentClusters(pc)) - clusters.addAll(c.getSearch().getClusters().values()); + for (ContentCluster c : getContentClusters(pc)) { + if (c.getSearch().hasIndexedCluster()) { + clusters.add(c.getSearch().getIndexed()); + } + } return clusters; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 4d6f454f7e0..ae9c0b681d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -31,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.Objects; import java.util.TreeMap; import java.util.function.Predicate; @@ -89,7 +87,8 @@ public class ContentSearchCluster extends TreeConfigProducer public Builder(Map documentDefinitions, Set globallyDistributedDocuments, - double fractionOfMemoryReserved, ResourceLimits resourceLimits) { + double fractionOfMemoryReserved, ResourceLimits resourceLimits) + { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.fractionOfMemoryReserved = fractionOfMemoryReserved; @@ -103,12 +102,9 @@ public class ContentSearchCluster extends TreeConfigProducer Boolean flushOnShutdownElem = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown"); Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog"); - var search = new ContentSearchCluster(ancestor, clusterName, - deployState.getProperties().featureFlags(), - documentDefinitions, - globallyDistributedDocuments, - getFlushOnShutdown(flushOnShutdownElem), - syncTransactionLog, + var search = new ContentSearchCluster(ancestor, clusterName, deployState.getProperties().featureFlags(), + documentDefinitions, globallyDistributedDocuments, + getFlushOnShutdown(flushOnShutdownElem), syncTransactionLog, fractionOfMemoryReserved); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); @@ -117,8 +113,7 @@ public class ContentSearchCluster extends TreeConfigProducer } search.setResourceLimits(resourceLimits); - buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); - buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); + buildSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -130,73 +125,18 @@ public class ContentSearchCluster extends TreeConfigProducer return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { - var schemas = deployState.getSchemas(); - for (var candidate : schemas) { - if (candidate.getName().equals(docTypeName)) { - return candidate; - } - } - return null; - } - - private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { + private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) return; - if (docElem == null) { - return; + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); } - for (ModelElement docType : docElem.subElements("document")) { - String docTypeName = docType.stringAttribute("type"); - String mode = docType.stringAttribute("mode"); - var schema = findResponsibleSchema(deployState, docTypeName); - if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { - buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); - } - } - } - - private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, - ContentSearchCluster search, ModelElement docType) { - String docTypeName = docType.stringAttribute("type"); - StreamingSearchCluster cluster = new StreamingSearchCluster(search, - clusterName + "." + docTypeName, - 0, - docTypeName, - clusterName); - search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); - } - - private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { - List indexedDefs = getIndexedSchemas(clusterElem); - if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); - } - - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); - } - } - - private List getIndexedSchemas(ModelElement clusterElem) { - List indexedDefs = new ArrayList<>(); - ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) { - return indexedDefs; - } - - for (ModelElement docType : docElem.subElements("document")) { - String mode = docType.stringAttribute("mode"); - if ("index".equals(mode)) { - indexedDefs.add(docType); - } - } - return indexedDefs; + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); } } @@ -247,9 +187,12 @@ public class ContentSearchCluster extends TreeConfigProducer cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - addCluster(cluster); + if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { + addCluster(cluster); + } } + private void addSchemas(DeployState deployState, List searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -257,6 +200,7 @@ public class ContentSearchCluster extends TreeConfigProducer if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); + if (schema.isDocumentsOnly()) continue; sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); } @@ -282,15 +226,9 @@ public class ContentSearchCluster extends TreeConfigProducer * with indexing, null if it has both or none. */ public Boolean isStreaming() { - boolean hasStreaming = false; - boolean hasIndexed = false; - for (var cluster : clusters.values()) { - if (cluster.hasStreaming()) - hasStreaming = true; - else - hasIndexed = true; - } - if (hasIndexed == hasStreaming) return null; + if (indexedCluster == null) return false; + boolean hasStreaming = indexedCluster.hasStreaming(); + if (indexedCluster.hasIndexed() == hasStreaming) return null; return hasStreaming; } @@ -356,21 +294,6 @@ public class ContentSearchCluster extends TreeConfigProducer this.redundancy = redundancy; } - private Optional findStreamingCluster(String docType) { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.schemas().get(docType) != null) - .findFirst(); - } - - public List getStreamingClusters() { - return getClusters().values().stream() - .filter(StreamingSearchCluster.class::isInstance) - .map(StreamingSearchCluster.class::cast) - .toList(); - } - public List getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); } public List getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); } public List getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); } @@ -382,13 +305,13 @@ public class ContentSearchCluster extends TreeConfigProducer } private boolean hasIndexingModeStreaming(NewDocumentType type) { - return findStreamingCluster(type.getFullName().getName()).isPresent(); + if (indexedCluster == null) return false; + return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING; } private boolean hasIndexingModeIndexed(NewDocumentType type) { - return !hasIndexingModeStreaming(type) - && hasIndexedCluster() - && getIndexed().hasDocumentDB(type.getFullName().getName()); + if (indexedCluster == null) return false; + return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX; } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -397,7 +320,7 @@ public class ContentSearchCluster extends TreeConfigProducer @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedCluster = false; + boolean hasAnyNonIndexedSchema = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -409,13 +332,13 @@ public class ContentSearchCluster extends TreeConfigProducer ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedCluster = true; - findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedSchema = true; + indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedCluster = true; + hasAnyNonIndexedSchema = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -424,7 +347,7 @@ public class ContentSearchCluster extends TreeConfigProducer builder.documentdb(ddbB); } - if (hasAnyNonIndexedCluster) { + if (hasAnyNonIndexedSchema) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); @@ -478,7 +401,6 @@ public class ContentSearchCluster extends TreeConfigProducer getIndexed().getConfig(builder); } } - public Map getClusters() { return clusters; } public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4a37b27d1c7..67d99e300de 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -45,9 +45,6 @@ public class IndexedSearchCluster extends SearchCluster implements summaryDecodePolicy = featureFlags.summaryDecodePolicy(); } - @Override - protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); } @@ -60,16 +57,6 @@ public class IndexedSearchCluster extends SearchCluster implements } public Tuning getTuning() { return tuning; } - @Override - public void deriveFromSchemas(DeployState deployState) { - for (SchemaInfo spec : schemas().values()) { - if (spec.fullSchema() instanceof DocumentOnlySchema) continue; - var db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode())); - add(db); - } - } - public void setSearchCoverage(SearchCoverage searchCoverage) { this.searchCoverage = searchCoverage; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index fc4b96ec384..9d459259253 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -3,18 +3,32 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -36,6 +50,50 @@ public abstract class SearchCluster extends TreeConfigProducer schemas = new LinkedHashMap<>(); private final Map documentDbs = new LinkedHashMap<>(); + private final Map documentDBProducerForStreaming = new HashMap<>(); + private final List legacyproxy = new ArrayList<>(); + + private static class LegacyStreamingProxy extends TreeConfigProducer implements + AttributesConfig.Producer, + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer, + JuniperrcConfig.Producer, + SummaryConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer + { + private final DocumentDatabase db; + LegacyStreamingProxy(TreeConfigProducer parent, String clusterName, + String schemaName, DerivedConfiguration derived) { + super(parent, "cluster." + clusterName + "." + schemaName); + this.db = new DocumentDatabase(this, schemaName, derived); + } + @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); } + } + + private static class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { + private final DerivedConfiguration derived; + + AttributesProducer(TreeConfigProducer parent, String docType, DerivedConfiguration derived) { + super(parent, docType); + this.derived = derived; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + derived.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } + } public SearchCluster(TreeConfigProducer parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -48,9 +106,6 @@ public abstract class SearchCluster extends TreeConfigProducer)getParent(); + documentDBProducerForStreaming.put(schemaName, new AttributesProducer(parent, schemaName, derived)); + legacyproxy.add(new LegacyStreamingProxy(parent, clusterName, schemaName, derived)); + } + } + } /** Returns the document databases contained in this cluster */ public List getDocumentDbs() { @@ -80,7 +147,6 @@ public abstract class SearchCluster extends TreeConfigProducer schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } @@ -91,7 +157,6 @@ public abstract class SearchCluster extends TreeConfigProducer parent, String clusterName, int index, - String docTypeName, String storageRouteSpec) { - super(parent, clusterName, index); - attributesConfig = new AttributesProducer(parent, docTypeName); - this.docTypeName = docTypeName; - this.storageRouteSpec = storageRouteSpec; - } - - @Override - protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } - public final String getStorageRouteSpec() { return storageRouteSpec; } - - public String getDocTypeName() { return docTypeName; } - - public DerivedConfiguration derived() { return db().getDerivedConfiguration(); } - - @Override - public void deriveFromSchemas(DeployState deployState) { - if (schemas().isEmpty()) return; - if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size()); - - Schema schema = schemas().values().stream().findAny().get().fullSchema(); - if ( ! schema.getName().equals(docTypeName)) - throw new IllegalArgumentException("Document type name '" + docTypeName + - "' must be the same as the schema name '" + schema.getName() + "'"); - add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING))); - } - - protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { - super.fillDocumentDBConfig(sdoc, ddbB); - ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up - } - - private DocumentDatabase db() { return getDocumentDbs().get(0); } - - // These are temporary until backend uses correct config id. - @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);} - - private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { - - AttributesProducer(TreeConfigProducer parent, String docType) { - super(parent, docType); - } - - @Override - public void getConfig(AttributesConfig.Builder builder) { - derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java index d3e62dae947..2879baf7b88 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/StreamingValidatorTest.java @@ -29,7 +29,7 @@ public class StreamingValidatorTest { new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/document_references_validation/") .create(); }); - assertTrue(exception.getMessage().contains("For search cluster 'content.ad', streaming schema 'ad': Attribute 'campaign_ref' has type 'Reference'. " + + assertTrue(exception.getMessage().contains("For search cluster 'content', streaming schema 'ad': Attribute 'campaign_ref' has type 'Reference'. " + "Document references and imported fields are not allowed in streaming search.")); } @@ -52,7 +52,7 @@ public class StreamingValidatorTest { "attribute { distance-metric: euclidean } }"); var warnings = filter(logger.warnings); assertEquals(1, warnings.size()); - assertEquals("For search cluster 'content.test', streaming schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting", + assertEquals("For search cluster 'content', streaming schema 'test', SD field 'nn': hnsw index is not relevant and not supported, ignoring setting", warnings.get(0)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java index ee64ceb6969..28eabd18539 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSchemaClusterChangeValidatorTest.java @@ -92,9 +92,9 @@ public class StreamingSchemaClusterChangeValidatorTest { private static final String ATTRIBUTE_INT_FIELD = "field f1 type int { indexing: attribute | summary }"; private static final String ATTRIBUTE_FAST_ACCESS_INT_FIELD = "field f1 type int { indexing: attribute | summary \n attribute: fast-access }"; private static final List FOO_SERVICE = - List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/0", "null")); + List.of(new ServiceInfo("searchnode", "null", null, null, "foo/search/cluster.foo/0", "null")); private static final List BAR_SERVICE = - List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/0", "null")); + List.of(new ServiceInfo("searchnode2", "null", null, null, "bar/search/cluster.bar/0", "null")); @Test void changing_field_type_requires_refeed() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index d3e69e64fe6..a4a5f4cb4df 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -63,7 +63,6 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s = a.getSearch(); assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); assertTrue(a.getPersistence() instanceof com.yahoo.vespa.model.content.engines.DummyPersistence.Factory); } @@ -83,7 +82,6 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s = a.getSearch(); assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); assertTrue(a.getPersistence() instanceof ProtonEngine.Factory); @@ -108,7 +106,6 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s = a.getSearch(); assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); assertTrue(a.getPersistence() instanceof ProtonEngine.Factory); @@ -130,7 +127,6 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s = a.getSearch(); assertFalse(s.hasIndexedCluster()); - assertTrue(s.getClusters().isEmpty()); assertNull(s.getIndexed()); assertNull(a.getRootGroup().getName()); @@ -151,7 +147,6 @@ public class ContentBuilderTest extends DomBuilderTest { ContentCluster c = CollectionUtil.first(m.getContentClusters().values()); ContentSearchCluster s = c.getSearch(); assertTrue(s.hasIndexedCluster()); - assertEquals(1, s.getClusters().size()); assertNotNull(s.getIndexed()); assertEquals("clu", s.getIndexed().getClusterName()); assertEquals(7.3, s.getIndexed().getQueryTimeout(), 0.0); @@ -196,11 +191,9 @@ public class ContentBuilderTest extends DomBuilderTest { ContentSearchCluster s; s = cluster.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertEquals(1, s.getClusters().size()); - assertNull(s.getIndexed()); - SearchCluster sc = s.getClusters().get(musicClusterId + ".music"); - assertEquals(musicClusterId + ".music", sc.getClusterName()); + assertTrue(s.hasIndexedCluster()); + SearchCluster sc = s.getIndexed(); + assertEquals(musicClusterId, sc.getClusterName()); assertEquals(musicClusterId, sc.getStorageRouteSpec()); assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory); @@ -236,24 +229,11 @@ public class ContentBuilderTest extends DomBuilderTest { " " + " " + ""); - ContentSearchCluster s; - - s = cluster.getSearch(); - assertFalse(s.hasIndexedCluster()); - assertEquals(2, s.getClusters().size()); - assertNull(s.getIndexed()); - { - String id = musicClusterId + ".book"; - SearchCluster sc = s.getClusters().get(id); - assertEquals(id, sc.getClusterName()); - assertEquals(musicClusterId, sc.getStorageRouteSpec()); - } - { - String id = musicClusterId + ".music"; - SearchCluster sc = s.getClusters().get(id); - assertEquals(id, sc.getClusterName()); - assertEquals(musicClusterId, sc.getStorageRouteSpec()); - } + ContentSearchCluster s = cluster.getSearch(); + assertTrue(s.hasIndexedCluster()); + assertEquals(2, s.getIndexed().getDocumentDbs().size()); + assertTrue(s.getIndexed().hasDocumentDB("book")); + assertTrue(s.getIndexed().hasDocumentDB("music")); assertTrue(cluster.getPersistence() instanceof ProtonEngine.Factory); assertEquals(1, cluster.getStorageCluster().getChildren().size()); @@ -277,7 +257,6 @@ public class ContentBuilderTest extends DomBuilderTest { s = b.getSearch(); assertTrue(s.hasIndexedCluster()); - assertEquals(1, s.getClusters().size()); assertNotNull(s.getIndexed()); assertEquals("b", s.getIndexed().getClusterName()); @@ -362,9 +341,7 @@ public class ContentBuilderTest extends DomBuilderTest { " " + " " + "", new TestProperties().setEnvironmentVariables(List.of("MY_1_ENV=xyz abc", "MY_2_ENV=2"))); - ContentSearchCluster s; - - s = b.getSearch(); + ContentSearchCluster s = b.getSearch(); assertTrue(s.hasIndexedCluster()); assertNotNull(s.getIndexed()); assertEquals(1, b.getStorageCluster().getChildren().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 8f02cf7a6d4..02a7a946e17 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; @@ -28,18 +27,13 @@ public class MockSearchClusters { @Override public void deriveFromSchemas(DeployState deployState) { } - @Override public List getDocumentDbs() {return List.of();} + @Override public List getDocumentDbs() { return List.of(); } @Override public void getConfig(AttributesConfig.Builder builder) {} @Override public void getConfig(DocumentdbInfoConfig.Builder builder) {} @Override public void getConfig(IndexInfoConfig.Builder builder) {} @Override public void getConfig(IlscriptsConfig.Builder builder) {} @Override public void getConfig(SchemaInfoConfig.Builder builder) {} - @Override - protected SearchCluster.IndexingMode getIndexingMode() { - return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME; - } - } public static SearchCluster mockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index 6f9127779db..ae08f78c404 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -163,7 +163,6 @@ public class IndexedTest extends ContentBaseTest { // "transactionlogserver"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); - assertFalse(s.getSearch().hasIndexedCluster()); StorServerConfig.Builder builder = new StorServerConfig.Builder(); s.getStorageCluster().getConfig(builder); @@ -175,7 +174,6 @@ public class IndexedTest extends ContentBaseTest { VespaModel model = getStreamingVespaModel(); ContentCluster s = model.getContentClusters().get("test"); assertNotNull(s); - assertFalse(s.getSearch().hasIndexedCluster()); ClusterListConfig config = model.getConfig(ClusterListConfig.class, VespaModel.ROOT_CONFIGID); assertEquals(1, config.storage().size()); assertEquals("test", config.storage(0).name()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index d3741405c15..b4c625a599b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -309,7 +309,7 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); var model = tester.createModelWithMode(mode, sds); - DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type"); + DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); assertEquals(1, dcfg.documentdb().size()); DocumentdbInfoConfig.Documentdb db = dcfg.documentdb(0); assertEquals("type", db.name()); @@ -328,13 +328,11 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); var model = tester.createModel(sds, ""); DocumentdbInfoConfig indexed_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test"); - assertEquals(1, indexed_cfg.documentdb().size()); + assertEquals(2, indexed_cfg.documentdb().size()); var db = indexed_cfg.documentdb(0); assertEquals("a", db.name()); assertEquals(DocumentdbInfoConfig.Documentdb.Mode.INDEX, db.mode()); - DocumentdbInfoConfig streaming_cfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.b"); - assertEquals(1, streaming_cfg.documentdb().size()); - db = streaming_cfg.documentdb(0); + db = indexed_cfg.documentdb(1); assertEquals("b", db.name()); assertEquals(DocumentdbInfoConfig.Documentdb.Mode.STREAMING, db.mode()); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index d61da0ebedf..d4e295fb51f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -55,7 +55,7 @@ public class GroupingValidator extends Searcher { var searchCluster = qrsConfig.searchcluster(clusterId); QrSearchersConfig.Searchcluster.Indexingmode.Enum indexingMode = searchCluster.indexingmode(); enabled = (indexingMode != QrSearchersConfig.Searchcluster.Indexingmode.STREAMING); - clusterName = enabled ? searchCluster.name() : null; + clusterName = searchCluster.name(); for (AttributesConfig.Attribute attr : attributesConfig.attribute()) { attributes.put(attr.name(), attr); } @@ -97,8 +97,7 @@ public class GroupingValidator extends Searcher { @Override public void visitExpression(GroupingExpression exp) { - if (exp instanceof AttributeMapLookupValue) { - AttributeMapLookupValue mapLookup = (AttributeMapLookupValue) exp; + if (exp instanceof AttributeMapLookupValue mapLookup) { verifyHasAttribute(mapLookup.getKeyAttribute()); verifyHasAttribute(mapLookup.getValueAttribute()); if (mapLookup.hasKeySourceAttribute()) { diff --git a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java index 033e567d53a..1c95a07cdfd 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/QueryValidator.java @@ -11,7 +11,6 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.schema.Field; -import com.yahoo.search.schema.FieldInfo; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; @@ -55,7 +54,7 @@ public class QueryValidator extends Searcher { public boolean visit(Item item) { if (item instanceof HasIndexItem indexItem) { var field = schema.fieldInfo(indexItem.getIndexName()); - if (! field.isPresent()) return true; + if (field.isEmpty()) return true; if (field.get().type().kind() == Field.Type.Kind.TENSOR) throw new IllegalArgumentException("Cannot search for terms in '" + indexItem.getIndexName() + "': It is a tensor field"); @@ -76,7 +75,7 @@ public class QueryValidator extends Searcher { if (schema.isStreaming()) return true; // prefix is always supported if (item instanceof PrefixItem prefixItem) { var field = schema.fieldInfo(prefixItem.getIndexName()); - if (! field.isPresent()) return true; + if (field.isEmpty()) return true; if ( ! field.get().isAttribute()) throw new IllegalArgumentException("'" + prefixItem.getIndexName() + "' is not an attribute field: Prefix matching is not supported"); if (field.get().isIndex()) // index overrides attribute -- cgit v1.2.3 From c809f7d0727f6dc833fd209f27e875e984c7da3c Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 15 Mar 2024 11:16:54 +0000 Subject: Provide all fields --- .../main/java/com/yahoo/schema/derived/DerivedConfiguration.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index a7f66966538..421f3d5a1d1 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -166,11 +166,7 @@ public class DerivedConfiguration { } public void getConfig(AttributesConfig.Builder builder) { - if (isStreaming()) { - getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } else { - getConfig(builder, AttributeFields.FieldSet.ALL); - } + getConfig(builder, AttributeFields.FieldSet.ALL); } public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { -- cgit v1.2.3 From 97754144fa0a7c5a61458ce2fe539ee377490c15 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 15 Mar 2024 13:00:57 +0100 Subject: Check for schemas with mixed mode. --- .../java/com/yahoo/vespa/model/content/ContentSearchCluster.java | 8 ++++++++ .../yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index ae9c0b681d2..01708333ed5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -232,6 +232,14 @@ public class ContentSearchCluster extends TreeConfigProducer return hasStreaming; } + public boolean hasStreaming() { + return (indexedCluster != null) && indexedCluster.hasStreaming(); + } + + public boolean hasIndexed() { + return (indexedCluster != null) && indexedCluster.hasIndexed(); + } + public List getSearchNodes() { return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 7fc713f81ef..f4e6244fa25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -13,7 +13,7 @@ public class EngineFactoryBuilder { public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) { ModelElement persistence = clusterElem.child("engine"); if (persistence != null) { - if (c.getSearch().hasIndexedCluster() && persistence.child("proton") == null) { + if (c.getSearch().hasIndexed() && persistence.child("proton") == null) { throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use as your engine."); } -- cgit v1.2.3