diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-03-15 11:56:34 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-15 11:56:34 +0100 |
commit | caa0e0bc8ba4632c953271ff281a02902d059114 (patch) | |
tree | c4411434f908907f62a77621b4b39c8dd57a097f /config-model/src/main/java | |
parent | 2946946a7a0b94dfeac971bb57942a7e9f7d2f9c (diff) | |
parent | 838476b8a9802aa3923c7f7a80fd6278e691b998 (diff) |
Merge pull request #30643 from vespa-engine/revert-30642-revert-30640-revert-30620-revert-30616-revert-30615-balder/single-searchcluster
Single searchcluster take 3
Diffstat (limited to 'config-model/src/main/java')
8 files changed, 284 insertions, 139 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index a7f66966538..421f3d5a1d1 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -166,11 +166,7 @@ public class DerivedConfiguration { } public void getConfig(AttributesConfig.Builder builder) { - if (isStreaming()) { - getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } else { - getConfig(builder, AttributeFields.FieldSet.ALL); - } + getConfig(builder, AttributeFields.FieldSet.ALL); } public void getConfig(AttributesConfig.Builder builder, AttributeFields.FieldSet fs) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 39894d95c2c..0d42219dade 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -30,11 +30,13 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { context.previousModel().getContentClusters().forEach((clusterName, currentCluster) -> { ContentCluster nextCluster = context.model().getContentClusters().get(clusterName); if (nextCluster != null) { - if (currentCluster.getSearch().getIndexed() != null && nextCluster.getSearch().getIndexed() != null) { - validateStreamingCluster(currentCluster, currentCluster.getSearch().getIndexed(), - nextCluster, nextCluster.getSearch().getIndexed()) - .forEach(context::require); - } + var nextStreamingClusters = nextCluster.getSearch().getClusters(); + currentCluster.getSearch().getClusters().values().forEach(currentStreamingCluster -> { + SearchCluster nextStreamingCluster = nextStreamingClusters.get(currentStreamingCluster.getClusterName()); + if (nextStreamingCluster != null) { + validateStreamingCluster(currentCluster, currentStreamingCluster, nextCluster, nextStreamingCluster).forEach(context::require); + } + }); } }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index a71d1025ece..91622e9fe31 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.Dispatcher; @@ -34,15 +35,15 @@ import static com.yahoo.vespa.model.container.PlatformBundles.SEARCH_AND_DOCPROC * @author gjoranv * @author Tony Vaagenes */ -public class ContainerSearch extends ContainerSubsystem<SearchChains> implements +public class ContainerSearch extends ContainerSubsystem<SearchChains> + implements IndexInfoConfig.Producer, IlscriptsConfig.Producer, QrSearchersConfig.Producer, QueryProfilesConfig.Producer, SemanticRulesConfig.Producer, PageTemplatesConfig.Producer, - SchemaInfoConfig.Producer -{ + SchemaInfoConfig.Producer { public static final String QUERY_PROFILE_REGISTRY_CLASS = CompiledQueryProfileRegistry.class.getName(); @@ -168,7 +169,25 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> implements @Override public void getConfig(QrSearchersConfig.Builder builder) { - searchClusters.forEach(sc -> builder.searchcluster(sc.getQrSearcherConfig())); + for (int i = 0; i < searchClusters.size(); i++) { + SearchCluster sys = findClusterWithId(searchClusters, i); + var scB = new QrSearchersConfig.Searchcluster.Builder().name(sys.getClusterName()); + for (SchemaInfo spec : sys.schemas().values()) { + scB.searchdef(spec.fullSchema().getName()); + } + scB.rankprofiles_configid(sys.getConfigId()); + scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); + scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(sys.getStorageRouteSpec())); + builder.searchcluster(scB); + } + } + + private static SearchCluster findClusterWithId(List<SearchCluster> clusters, int index) { + for (SearchCluster sys : clusters) { + if (sys.getClusterIndex() == index) + return sys; + } + throw new IllegalArgumentException("No search cluster with index " + index + " exists"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 4ac7a8e442a..1254f8e110a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -135,11 +135,8 @@ public class Content extends ConfigModel { public static List<SearchCluster> getSearchClusters(ConfigModelRepo pc) { List<SearchCluster> clusters = new ArrayList<>(); - for (ContentCluster c : getContentClusters(pc)) { - if (c.getSearch().hasIndexedCluster()) { - clusters.add(c.getSearch().getIndexed()); - } - } + for (ContentCluster c : getContentClusters(pc)) + clusters.addAll(c.getSearch().getClusters().values()); return clusters; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index ae9c0b681d2..4d6f454f7e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.LinkedHashMap; +import java.util.Optional; import java.util.Objects; import java.util.TreeMap; import java.util.function.Predicate; @@ -87,8 +89,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public Builder(Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - double fractionOfMemoryReserved, ResourceLimits resourceLimits) - { + double fractionOfMemoryReserved, ResourceLimits resourceLimits) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.fractionOfMemoryReserved = fractionOfMemoryReserved; @@ -102,9 +103,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> Boolean flushOnShutdownElem = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown"); Boolean syncTransactionLog = clusterElem.childAsBoolean("engine.proton.sync-transactionlog"); - var search = new ContentSearchCluster(ancestor, clusterName, deployState.getProperties().featureFlags(), - documentDefinitions, globallyDistributedDocuments, - getFlushOnShutdown(flushOnShutdownElem), syncTransactionLog, + var search = new ContentSearchCluster(ancestor, clusterName, + deployState.getProperties().featureFlags(), + documentDefinitions, + globallyDistributedDocuments, + getFlushOnShutdown(flushOnShutdownElem), + syncTransactionLog, fractionOfMemoryReserved); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); @@ -113,7 +117,8 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } search.setResourceLimits(resourceLimits); - buildSearchCluster(deployState, clusterElem, clusterName, search); + buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); + buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); return search; } @@ -125,18 +130,73 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> return clusterElem.childAsDouble("engine.proton.query-timeout"); } - private void buildSearchCluster(DeployState deployState, ModelElement clusterElem, - String clusterName, ContentSearchCluster search) { + private static Schema findResponsibleSchema(DeployState deployState, String docTypeName) { + var schemas = deployState.getSchemas(); + for (var candidate : schemas) { + if (candidate.getName().equals(docTypeName)) { + return candidate; + } + } + return null; + } + + private void buildAllStreamingSearchClusters(DeployState deployState, ModelElement clusterElem, String clusterName, ContentSearchCluster search) { ModelElement docElem = clusterElem.child("documents"); - if (docElem == null) return; - Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); - if (visibilityDelay != null) { - search.setVisibilityDelay(visibilityDelay); + if (docElem == null) { + return; } - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); - search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), docElem.subElements("document")); + for (ModelElement docType : docElem.subElements("document")) { + String docTypeName = docType.stringAttribute("type"); + String mode = docType.stringAttribute("mode"); + var schema = findResponsibleSchema(deployState, docTypeName); + if ("streaming".equals(mode) && schema != null && !schema.isDocumentsOnly()) { + buildStreamingSearchCluster(deployState, clusterElem, clusterName, search, docType); + } + } + } + + private void buildStreamingSearchCluster(DeployState deployState, ModelElement clusterElem, String clusterName, + ContentSearchCluster search, ModelElement docType) { + String docTypeName = docType.stringAttribute("type"); + StreamingSearchCluster cluster = new StreamingSearchCluster(search, + clusterName + "." + docTypeName, + 0, + docTypeName, + clusterName); + search.addSearchCluster(deployState, cluster, getQueryTimeout(clusterElem), List.of(docType)); + } + + private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem, + String clusterName, ContentSearchCluster search) { + List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); + if (!indexedDefs.isEmpty()) { + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, search, deployState.featureFlags()); + + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + search.setVisibilityDelay(visibilityDelay); + } + + search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs); + } + } + + private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) { + List<ModelElement> indexedDefs = new ArrayList<>(); + ModelElement docElem = clusterElem.child("documents"); + if (docElem == null) { + return indexedDefs; + } + + for (ModelElement docType : docElem.subElements("document")) { + String mode = docType.stringAttribute("mode"); + if ("index".equals(mode)) { + indexedDefs.add(docType); + } + } + return indexedDefs; } } @@ -187,12 +247,9 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> cluster.setQueryTimeout(queryTimeout); } cluster.deriveFromSchemas(deployState); - if ( ! cluster.schemas().values().stream().allMatch(schemaInfo -> schemaInfo.getIndexMode() == SchemaInfo.IndexMode.STORE_ONLY)) { - addCluster(cluster); - } + addCluster(cluster); } - private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, SearchCluster sc) { for (ModelElement e : searchDefs) { SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e); @@ -200,7 +257,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (schema == null) throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); - if (schema.isDocumentsOnly()) continue; sc.add(new SchemaInfo(schema, e.stringAttribute("mode"), deployState.rankProfileRegistry(), null)); } @@ -226,9 +282,15 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> * with indexing, null if it has both or none. */ public Boolean isStreaming() { - if (indexedCluster == null) return false; - boolean hasStreaming = indexedCluster.hasStreaming(); - if (indexedCluster.hasIndexed() == hasStreaming) return null; + boolean hasStreaming = false; + boolean hasIndexed = false; + for (var cluster : clusters.values()) { + if (cluster.hasStreaming()) + hasStreaming = true; + else + hasIndexed = true; + } + if (hasIndexed == hasStreaming) return null; return hasStreaming; } @@ -294,6 +356,21 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> this.redundancy = redundancy; } + private Optional<StreamingSearchCluster> findStreamingCluster(String docType) { + return getClusters().values().stream() + .filter(StreamingSearchCluster.class::isInstance) + .map(StreamingSearchCluster.class::cast) + .filter(ssc -> ssc.schemas().get(docType) != null) + .findFirst(); + } + + public List<StreamingSearchCluster> getStreamingClusters() { + return getClusters().values().stream() + .filter(StreamingSearchCluster.class::isInstance) + .map(StreamingSearchCluster.class::cast) + .toList(); + } + public List<NewDocumentType> getDocumentTypesWithStreamingCluster() { return documentTypes(this::hasIndexingModeStreaming); } public List<NewDocumentType> getDocumentTypesWithIndexedCluster() { return documentTypes(this::hasIndexingModeIndexed); } public List<NewDocumentType> getDocumentTypesWithStoreOnly() { return documentTypes(this::hasIndexingModeStoreOnly); } @@ -305,13 +382,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> } private boolean hasIndexingModeStreaming(NewDocumentType type) { - if (indexedCluster == null) return false; - return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.STREAMING; + return findStreamingCluster(type.getFullName().getName()).isPresent(); } private boolean hasIndexingModeIndexed(NewDocumentType type) { - if (indexedCluster == null) return false; - return indexedCluster.schemas().get(type.getName()).getIndexMode() == SchemaInfo.IndexMode.INDEX; + return !hasIndexingModeStreaming(type) + && hasIndexedCluster() + && getIndexed().hasDocumentDB(type.getFullName().getName()); } private boolean hasIndexingModeStoreOnly(NewDocumentType type) { @@ -320,7 +397,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public void getConfig(ProtonConfig.Builder builder) { - boolean hasAnyNonIndexedSchema = false; + boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -332,13 +409,13 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { - hasAnyNonIndexedSchema = true; - indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); + hasAnyNonIndexedCluster = true; + findStreamingCluster(docTypeName).get().fillDocumentDBConfig(type.getFullName().getName(), ddbB); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { - indexedCluster.fillDocumentDBConfig(type.getFullName().getName(), ddbB); + getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); } else { - hasAnyNonIndexedSchema = true; + hasAnyNonIndexedCluster = true; ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { @@ -347,7 +424,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> builder.documentdb(ddbB); } - if (hasAnyNonIndexedSchema) { + if (hasAnyNonIndexedCluster) { builder.feeding.concurrency(Math.min(1.0, defaultFeedConcurrency*2)); } else { builder.feeding.concurrency(defaultFeedConcurrency); @@ -401,6 +478,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> getIndexed().getConfig(builder); } } + public Map<String, SearchCluster> getClusters() { return clusters; } public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 67d99e300de..4a37b27d1c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -45,6 +45,9 @@ public class IndexedSearchCluster extends SearchCluster implements summaryDecodePolicy = featureFlags.summaryDecodePolicy(); } + @Override + protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } + public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); } @@ -57,6 +60,16 @@ public class IndexedSearchCluster extends SearchCluster implements } public Tuning getTuning() { return tuning; } + @Override + public void deriveFromSchemas(DeployState deployState) { + for (SchemaInfo spec : schemas().values()) { + if (spec.fullSchema() instanceof DocumentOnlySchema) continue; + var db = new DocumentDatabase(this, spec.fullSchema().getName(), + new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode())); + add(db); + } + } + public void setSearchCoverage(SearchCoverage searchCoverage) { this.searchCoverage = searchCoverage; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 9d459259253..fc4b96ec384 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -3,32 +3,18 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.schema.DocumentOnlySchema; -import com.yahoo.schema.derived.AttributeFields; -import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.SummaryConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.config.search.summary.JuniperrcConfig; -import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; -import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -50,50 +36,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer private Double visibilityDelay = 0.0; private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>(); private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); - private final Map<String, AttributesProducer> documentDBProducerForStreaming = new HashMap<>(); - private final List<LegacyStreamingProxy> legacyproxy = new ArrayList<>(); - - private static class LegacyStreamingProxy extends TreeConfigProducer<AnyConfigProducer> implements - AttributesConfig.Producer, - RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - RankingExpressionsConfig.Producer, - OnnxModelsConfig.Producer, - JuniperrcConfig.Producer, - SummaryConfig.Producer, - VsmsummaryConfig.Producer, - VsmfieldsConfig.Producer - { - private final DocumentDatabase db; - LegacyStreamingProxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, - String schemaName, DerivedConfiguration derived) { - super(parent, "cluster." + clusterName + "." + schemaName); - this.db = new DocumentDatabase(this, schemaName, derived); - } - @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); } - @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); } - } - - private static class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { - private final DerivedConfiguration derived; - - AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType, DerivedConfiguration derived) { - super(parent, docType); - this.derived = derived; - } - - @Override - public void getConfig(AttributesConfig.Builder builder) { - derived.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } - } public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -106,6 +48,9 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public void add(SchemaInfo schema) { schemas.put(schema.name(), schema); } + public void add(DocumentDatabase db) { + documentDbs.put(db.getName(), db); + } public boolean hasDocumentDB(String name) { return documentDbs.containsKey(name); @@ -127,19 +72,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer * Derives the schemas from the application package. * Also stores the document names contained in the schemas. */ - public void deriveFromSchemas(DeployState deployState) { - for (SchemaInfo spec : schemas().values()) { - if (spec.fullSchema() instanceof DocumentOnlySchema) continue; // TODO verify if this special handling is necessary - String schemaName = spec.fullSchema().getName(); - var derived = new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode()); - documentDbs.put(schemaName, new DocumentDatabase(this, schemaName, derived)); - if (spec.getIndexMode() == SchemaInfo.IndexMode.STREAMING) { - var parent = (TreeConfigProducer<AnyConfigProducer>)getParent(); - documentDBProducerForStreaming.put(schemaName, new AttributesProducer(parent, schemaName, derived)); - legacyproxy.add(new LegacyStreamingProxy(parent, clusterName, schemaName, derived)); - } - } - } + public abstract void deriveFromSchemas(DeployState deployState); /** Returns the document databases contained in this cluster */ public List<DocumentDatabase> getDocumentDbs() { @@ -147,6 +80,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer } public String getClusterName() { return clusterName; } + public final String getIndexingModeName() { return getIndexingMode().getName(); } public final boolean hasStreaming() { return schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } @@ -157,6 +91,7 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public final void setQueryTimeout(Double to) { this.queryTimeout = to; } public final void setVisibilityDelay(double delay) { this.visibilityDelay = delay; } + protected abstract IndexingMode getIndexingMode(); public final Double getVisibilityDelay() { return visibilityDelay; } public final Double getQueryTimeout() { return queryTimeout; } public final void setClusterIndex(int index) { this.index = index; } @@ -165,26 +100,13 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) { DocumentDatabase db = documentDbs.get(documentType); if (db != null) { - builder.inputdoctypename(documentType); - if (db.getDerivedConfiguration().isStreaming()) { - builder.configid(documentDBProducerForStreaming.get(documentType).getConfigId()); - } else { - builder.configid(db.getConfigId()); - } + fillDocumentDBConfig(db, builder); } } - public QrSearchersConfig.Searchcluster.Builder getQrSearcherConfig() { - var builder = new QrSearchersConfig.Searchcluster.Builder() - .name(getClusterName()) - .rankprofiles_configid(getConfigId()) - .storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(getStorageRouteSpec())) - .indexingmode(hasStreaming() ? QrSearchersConfig.Searchcluster.Indexingmode.STREAMING - : QrSearchersConfig.Searchcluster.Indexingmode.REALTIME); - for (SchemaInfo spec : schemas().values()) { - builder.searchdef(spec.fullSchema().getName()); - } - return builder; + protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { + ddbB.inputdoctypename(sdoc.getSchemaName()) + .configid(sdoc.getConfigId()); } @Override @@ -220,6 +142,24 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer @Override public String toString() { return "search-capable cluster '" + clusterName + "'"; } + public static final class IndexingMode { + + public static final IndexingMode REALTIME = new IndexingMode("REALTIME"); + public static final IndexingMode STREAMING = new IndexingMode("STREAMING"); + + private final String name; + + private IndexingMode(String name) { + this.name = name; + } + + public String getName() { return name; } + + public String toString() { + return "indexingmode: " + name; + } + } + /** * Class used to retrieve combined configuration from multiple document databases. * It is not a direct {@link ConfigInstance.Producer} of those configs, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java new file mode 100644 index 00000000000..e188a086614 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -0,0 +1,100 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; + +/** + * A search cluster of type streaming. + * + * @author baldersheim + * @author vegardh + */ +public class StreamingSearchCluster extends SearchCluster implements + AttributesConfig.Producer, + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer, + JuniperrcConfig.Producer, + SummaryConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer +{ + private final String storageRouteSpec; + private final AttributesProducer attributesConfig; + private final String docTypeName; + + public StreamingSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, + String docTypeName, String storageRouteSpec) { + super(parent, clusterName, index); + attributesConfig = new AttributesProducer(parent, docTypeName); + this.docTypeName = docTypeName; + this.storageRouteSpec = storageRouteSpec; + } + + @Override + protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } + public final String getStorageRouteSpec() { return storageRouteSpec; } + + public String getDocTypeName() { return docTypeName; } + + public DerivedConfiguration derived() { return db().getDerivedConfiguration(); } + + @Override + public void deriveFromSchemas(DeployState deployState) { + if (schemas().isEmpty()) return; + if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size()); + + Schema schema = schemas().values().stream().findAny().get().fullSchema(); + if ( ! schema.getName().equals(docTypeName)) + throw new IllegalArgumentException("Document type name '" + docTypeName + + "' must be the same as the schema name '" + schema.getName() + "'"); + add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING))); + } + + protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { + super.fillDocumentDBConfig(sdoc, ddbB); + ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up + } + + private DocumentDatabase db() { return getDocumentDbs().get(0); } + + // These are temporary until backend uses correct config id. + @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);} + + private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { + + AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType) { + super(parent, docType); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } + } + +} |