diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/vespa/model/search')
3 files changed, 112 insertions, 144 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4a37b27d1c7..c9c7e424ee6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -2,12 +2,8 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.schema.DocumentOnlySchema; -import com.yahoo.schema.derived.DerivedConfiguration; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; import com.yahoo.vespa.config.search.DispatchNodesConfig; @@ -45,9 +41,6 @@ public class IndexedSearchCluster extends SearchCluster implements summaryDecodePolicy = featureFlags.summaryDecodePolicy(); } - @Override - protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); } @@ -60,16 +53,6 @@ public class IndexedSearchCluster extends SearchCluster implements } public Tuning getTuning() { return tuning; } - @Override - public void deriveFromSchemas(DeployState deployState) { - for (SchemaInfo spec : schemas().values()) { - if (spec.fullSchema() instanceof DocumentOnlySchema) continue; - var db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode())); - add(db); - } - } - public void setSearchCoverage(SearchCoverage searchCoverage) { this.searchCoverage = searchCoverage; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index fc4b96ec384..8a60930664e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -3,18 +3,33 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -36,6 +51,50 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer private Double visibilityDelay = 0.0; private final Map<String, SchemaInfo> schemas = new LinkedHashMap<>(); private final Map<String, DocumentDatabase> documentDbs = new LinkedHashMap<>(); + private final Map<String, AttributesProducer> documentDBProducerForStreaming = new HashMap<>(); + private final List<LegacyStreamingProxy> legacyproxy = new ArrayList<>(); + + private static class LegacyStreamingProxy extends TreeConfigProducer<AnyConfigProducer> implements + AttributesConfig.Producer, + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer, + JuniperrcConfig.Producer, + SummaryConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer + { + private final DocumentDatabase db; + LegacyStreamingProxy(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, + String schemaName, DerivedConfiguration derived) { + super(parent, "cluster." + clusterName + "." + schemaName); + this.db = new DocumentDatabase(this, schemaName, derived); + } + @Override public void getConfig(SummaryConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(AttributesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(OnnxModelsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingConstantsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankProfilesConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(JuniperrcConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmfieldsConfig.Builder builder) { db.getConfig(builder); } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { db.getConfig(builder); } + } + + private static class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { + private final DerivedConfiguration derived; + + AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType, DerivedConfiguration derived) { + super(parent, docType); + this.derived = derived; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + derived.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); + } + } public SearchCluster(TreeConfigProducer<?> parent, String clusterName, int index) { super(parent, "cluster." + clusterName); @@ -48,9 +107,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public void add(SchemaInfo schema) { schemas.put(schema.name(), schema); } - public void add(DocumentDatabase db) { - documentDbs.put(db.getName(), db); - } public boolean hasDocumentDB(String name) { return documentDbs.containsKey(name); @@ -72,7 +128,19 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer * Derives the schemas from the application package. * Also stores the document names contained in the schemas. */ - public abstract void deriveFromSchemas(DeployState deployState); + public void deriveFromSchemas(DeployState deployState) { + for (SchemaInfo spec : schemas().values()) { + if (spec.fullSchema() instanceof DocumentOnlySchema) continue; // TODO verify if this special handling is necessary + String schemaName = spec.fullSchema().getName(); + var derived = new DerivedConfiguration(deployState, spec.fullSchema(), spec.getIndexMode()); + documentDbs.put(schemaName, new DocumentDatabase(this, schemaName, derived)); + if (spec.getIndexMode() == SchemaInfo.IndexMode.STREAMING) { + var parent = (TreeConfigProducer<AnyConfigProducer>)getParent(); + documentDBProducerForStreaming.put(schemaName, new AttributesProducer(parent, schemaName, derived)); + legacyproxy.add(new LegacyStreamingProxy(parent, clusterName, schemaName, derived)); + } + } + } /** Returns the document databases contained in this cluster */ public List<DocumentDatabase> getDocumentDbs() { @@ -80,7 +148,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer } public String getClusterName() { return clusterName; } - public final String getIndexingModeName() { return getIndexingMode().getName(); } public final boolean hasStreaming() { return schemas().values().stream().anyMatch(schema -> schema.getIndexMode() == SchemaInfo.IndexMode.STREAMING); } @@ -91,7 +158,6 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public final void setQueryTimeout(Double to) { this.queryTimeout = to; } public final void setVisibilityDelay(double delay) { this.visibilityDelay = delay; } - protected abstract IndexingMode getIndexingMode(); public final Double getVisibilityDelay() { return visibilityDelay; } public final Double getQueryTimeout() { return queryTimeout; } public final void setClusterIndex(int index) { this.index = index; } @@ -100,13 +166,26 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) { DocumentDatabase db = documentDbs.get(documentType); if (db != null) { - fillDocumentDBConfig(db, builder); + builder.inputdoctypename(documentType); + if (db.getDerivedConfiguration().isStreaming()) { + builder.configid(documentDBProducerForStreaming.get(documentType).getConfigId()); + } else { + builder.configid(db.getConfigId()); + } } } - protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { - ddbB.inputdoctypename(sdoc.getSchemaName()) - .configid(sdoc.getConfigId()); + public QrSearchersConfig.Searchcluster.Builder getQrSearcherConfig() { + var builder = new QrSearchersConfig.Searchcluster.Builder() + .name(getClusterName()) + .rankprofiles_configid(getConfigId()) + .storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().routespec(getStorageRouteSpec())) + .indexingmode(hasStreaming() ? QrSearchersConfig.Searchcluster.Indexingmode.STREAMING + : QrSearchersConfig.Searchcluster.Indexingmode.REALTIME); + for (SchemaInfo spec : schemas().values()) { + builder.searchdef(spec.fullSchema().getName()); + } + return builder; } @Override @@ -139,27 +218,33 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer new Join(documentDbs.values()).getConfig(builder); } - @Override - public String toString() { return "search-capable cluster '" + clusterName + "'"; } - - public static final class IndexingMode { - - public static final IndexingMode REALTIME = new IndexingMode("REALTIME"); - public static final IndexingMode STREAMING = new IndexingMode("STREAMING"); - - private final String name; - - private IndexingMode(String name) { - this.name = name; + public void getConfig(ClusterConfig.Builder builder) { + builder.clusterId(getClusterIndex()); + builder.clusterName(getClusterName()); + builder.storageRoute(getClusterName()); + builder.configid(getConfigId()); + if (visibilityDelay != null) { + builder.cacheTimeout(convertVisibilityDelay(visibilityDelay)); } - - public String getName() { return name; } - - public String toString() { - return "indexingmode: " + name; + if (hasStreaming()) { + builder.indexMode(ClusterConfig.IndexMode.Enum.STREAMING); + } else { + builder.indexMode(ClusterConfig.IndexMode.Enum.INDEX); } } + // The semantics of visibility delay in search is deactivating caches if the + // delay is less than 1.0, in qrs the cache is deactivated if the delay is 0 + // (or less). 1.0 seems a little arbitrary, so just doing the conversion + // here instead of having two totally independent implementations having to + // follow each other down in the modules. + private static Double convertVisibilityDelay(Double visibilityDelay) { + return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay; + } + + @Override + public String toString() { return "search-capable cluster '" + clusterName + "'"; } + /** * Class used to retrieve combined configuration from multiple document databases. * It is not a direct {@link ConfigInstance.Producer} of those configs, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java deleted file mode 100644 index e188a086614..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.schema.Schema; -import com.yahoo.schema.derived.AttributeFields; -import com.yahoo.schema.derived.DerivedConfiguration; -import com.yahoo.schema.derived.SchemaInfo; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.SummaryConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.config.search.summary.JuniperrcConfig; -import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; -import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; - -/** - * A search cluster of type streaming. - * - * @author baldersheim - * @author vegardh - */ -public class StreamingSearchCluster extends SearchCluster implements - AttributesConfig.Producer, - RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - RankingExpressionsConfig.Producer, - OnnxModelsConfig.Producer, - JuniperrcConfig.Producer, - SummaryConfig.Producer, - VsmsummaryConfig.Producer, - VsmfieldsConfig.Producer -{ - private final String storageRouteSpec; - private final AttributesProducer attributesConfig; - private final String docTypeName; - - public StreamingSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, - String docTypeName, String storageRouteSpec) { - super(parent, clusterName, index); - attributesConfig = new AttributesProducer(parent, docTypeName); - this.docTypeName = docTypeName; - this.storageRouteSpec = storageRouteSpec; - } - - @Override - protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } - public final String getStorageRouteSpec() { return storageRouteSpec; } - - public String getDocTypeName() { return docTypeName; } - - public DerivedConfiguration derived() { return db().getDerivedConfiguration(); } - - @Override - public void deriveFromSchemas(DeployState deployState) { - if (schemas().isEmpty()) return; - if (schemas().size() > 1) throw new IllegalArgumentException("Only a single schema is supported, got " + schemas().size()); - - Schema schema = schemas().values().stream().findAny().get().fullSchema(); - if ( ! schema.getName().equals(docTypeName)) - throw new IllegalArgumentException("Document type name '" + docTypeName + - "' must be the same as the schema name '" + schema.getName() + "'"); - add(new DocumentDatabase(this, docTypeName, new DerivedConfiguration(deployState, schema, SchemaInfo.IndexMode.STREAMING))); - } - - protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { - super.fillDocumentDBConfig(sdoc, ddbB); - ddbB.configid(attributesConfig.getConfigId()); // Temporary until fully cleaned up - } - - private DocumentDatabase db() { return getDocumentDbs().get(0); } - - // These are temporary until backend uses correct config id. - @Override public void getConfig(SummaryConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankingConstantsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(RankingExpressionsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(JuniperrcConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(VsmfieldsConfig.Builder builder) { db().getConfig(builder); } - @Override public void getConfig(VsmsummaryConfig.Builder builder) { db().getConfig(builder);} - - private class AttributesProducer extends AnyConfigProducer implements AttributesConfig.Producer { - - AttributesProducer(TreeConfigProducer<AnyConfigProducer> parent, String docType) { - super(parent, docType); - } - - @Override - public void getConfig(AttributesConfig.Builder builder) { - derived().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } - } - -} |