diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-05-31 16:36:40 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-31 16:36:40 +0200 |
commit | 00e2eae6db19c374a288e239a9df3f96e35cce77 (patch) | |
tree | 940c99100a445d9424d2b06037865c9dd67783c3 | |
parent | dc6437be5ca9e522004e2d01bc5df9dedd8d00cf (diff) | |
parent | 81f2213e73a0e8819f8933fd0304e36f677f0a00 (diff) |
Merge pull request #22800 from vespa-engine/bratseth/schema-info-take-2
Bratseth/schema info take 2
7 files changed, 79 insertions, 97 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 458f8dfbd68..773d696f3e8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -11,6 +11,7 @@ import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.MatchType; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; +import com.yahoo.vespa.model.search.StreamingSearchCluster; import java.util.List; import java.util.logging.Level; @@ -25,21 +26,21 @@ public class StreamingValidator extends Validator { List<SearchCluster> searchClusters = model.getSearchClusters(); for (SearchCluster cluster : searchClusters) { if ( ! cluster.isStreaming()) continue; - warnStreamingAttributes(cluster, deployState.getDeployLogger()); - warnStreamingGramMatching(cluster, deployState.getDeployLogger()); - failStreamingDocumentReferences(cluster); + + var streamingCluster = (StreamingSearchCluster)cluster; + warnStreamingAttributes(streamingCluster, deployState.getDeployLogger()); + warnStreamingGramMatching(streamingCluster, deployState.getDeployLogger()); + failStreamingDocumentReferences(streamingCluster); } } - private static void warnStreamingGramMatching(SearchCluster sc, DeployLogger logger) { - if (sc.getSchemaConfig() != null) { - for (ImmutableSDField sd : sc.getSchemaConfig().getSchema().allConcreteFields()) { - if (sd.getMatching().getType() == MatchType.GRAM) { - logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + - sc.getClusterName() + - "', SD field '" + sd.getName() + - "': n-gram matching is not supported for streaming search."); - } + private static void warnStreamingGramMatching(StreamingSearchCluster sc, DeployLogger logger) { + for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { + if (sd.getMatching().getType() == MatchType.GRAM) { + logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + + sc.getClusterName() + + "', SD field '" + sd.getName() + + "': n-gram matching is not supported for streaming search."); } } } @@ -50,17 +51,15 @@ public class StreamingValidator extends Validator { * @param sc a search cluster to be checked for attributes in streaming search * @param logger a DeployLogger */ - private static void warnStreamingAttributes(SearchCluster sc, DeployLogger logger) { - if (sc.getSchemaConfig() != null) { - for (ImmutableSDField sd : sc.getSchemaConfig().getSchema().allConcreteFields()) { - if (sd.doesAttributing()) { - warnStreamingAttribute(sc, sd, logger); - } + private static void warnStreamingAttributes(StreamingSearchCluster sc, DeployLogger logger) { + for (ImmutableSDField sd : sc.derived().getSchema().allConcreteFields()) { + if (sd.doesAttributing()) { + warnStreamingAttribute(sc, sd, logger); } } } - private static void warnStreamingAttribute(SearchCluster sc, ImmutableSDField sd, DeployLogger logger) { + private static void warnStreamingAttribute(StreamingSearchCluster sc, ImmutableSDField sd, DeployLogger logger) { // If the field is numeric, we can't print this, because we may have converted the field to // attribute indexing ourselves (IntegerIndex2Attribute) if (sd.getDataType() instanceof NumericDataType) return; @@ -69,8 +68,8 @@ public class StreamingValidator extends Validator { "': 'attribute' has same match semantics as 'index'."); } - private static void failStreamingDocumentReferences(SearchCluster sc) { - for (Attribute attribute : sc.getSchemaConfig().getAttributeFields().attributes()) { + private static void failStreamingDocumentReferences(StreamingSearchCluster sc) { + for (Attribute attribute : sc.derived().getAttributeFields().attributes()) { DataType dataType = attribute.getDataType(); if (dataType instanceof NewDocumentReferenceDataType) { String errorMessage = String.format("For streaming search cluster '%s': Attribute '%s' has type '%s'. " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 0563aa02149..167aff1064b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -64,11 +64,11 @@ public class StreamingSearchClusterChangeValidator implements ChangeValidator { getDocumentType(currentCluster, currentStreamingCluster), getDocumentType(nextCluster, nextStreamingCluster), overrides, now)); result.addAll(validateAttributeFastAccessAdded(currentCluster.id(), - currentStreamingCluster.getSchemaConfig().getAttributeFields(), - nextStreamingCluster.getSchemaConfig().getAttributeFields())); + currentStreamingCluster.derived().getAttributeFields(), + nextStreamingCluster.derived().getAttributeFields())); result.addAll(validateAttributeFastAccessRemoved(currentCluster.id(), - currentStreamingCluster.getSchemaConfig().getAttributeFields(), - nextStreamingCluster.getSchemaConfig().getAttributeFields())); + currentStreamingCluster.derived().getAttributeFields(), + nextStreamingCluster.derived().getAttributeFields())); return modifyActions(result, getSearchNodeServices(nextCluster), nextStreamingCluster.getDocTypeName()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 4be5153293f..1ab5fd22648 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -329,7 +329,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> return getClusters().values().stream() .filter(StreamingSearchCluster.class::isInstance) .map(StreamingSearchCluster.class::cast) - .filter(ssc -> ssc.getSchemaConfig().getSchema().getName().equals(docType)) + .filter(ssc -> ssc.schemas().get(docType) != null) .findFirst(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index d63ae451f39..9efe61d134f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -222,9 +222,6 @@ public class IndexedSearchCluster extends SearchCluster } @Override - public DerivedConfiguration getSchemaConfig() { return null; } - - @Override public void getConfig(IndexInfoConfig.Builder builder) { documentDbsConfigProducer.getConfig(builder); } @@ -239,12 +236,10 @@ public class IndexedSearchCluster extends SearchCluster documentDbsConfigProducer.getConfig(builder); } - @Override public void getConfig(AttributesConfig.Builder builder) { documentDbsConfigProducer.getConfig(builder); } - @Override public void getConfig(RankProfilesConfig.Builder builder) { documentDbsConfigProducer.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index d727e6d6395..f033399c787 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -5,14 +5,13 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.schema.derived.SummaryMap; +import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.search.config.IndexInfoConfig; -import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.config.model.producer.AbstractConfigProducer; import java.util.Collections; @@ -147,35 +146,10 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster public final int getClusterIndex() { return index; } public abstract void defaultDocumentsConfig(); - public abstract DerivedConfiguration getSchemaConfig(); - - // TODO: The get methods below should be moved to StreamingSearchCluster - - @Override - public void getConfig(IndexInfoConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getIndexInfo().getConfig(builder); - } - - @Override - public void getConfig(SchemaInfoConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getSchemaInfo().getConfig(builder); - } - - @Override - public void getConfig(IlscriptsConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getIndexingScript().getConfig(builder); - } - public void getConfig(AttributesConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getConfig(builder); - } - - public void getConfig(RankProfilesConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getRankProfileList().getConfig(builder); - } + public abstract void getConfig(AttributesConfig.Builder builder); - @Override - public abstract void getConfig(DocumentdbInfoConfig.Builder builder); + public abstract void getConfig(RankProfilesConfig.Builder builder); @Override public String toString() { return "search-capable cluster '" + clusterName + "'"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 724ba70510f..79adfb2ea13 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -7,12 +7,15 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.schema.Schema; import com.yahoo.schema.derived.AttributeFields; import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; /** * A search cluster of type streaming. @@ -20,8 +23,8 @@ import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; * @author baldersheim * @author vegardh */ -public class StreamingSearchCluster extends SearchCluster implements - DocumentdbInfoConfig.Producer, +public class StreamingSearchCluster extends SearchCluster implements + DocumentdbInfoConfig.Producer, RankProfilesConfig.Producer, VsmsummaryConfig.Producer, VsmfieldsConfig.Producer, @@ -31,7 +34,7 @@ public class StreamingSearchCluster extends SearchCluster implements private final String storageRouteSpec; private final AttributesProducer attributesConfig; private final String docTypeName; - private DerivedConfiguration schemaConfig = null; + private DerivedConfiguration derivedConfig = null; public StreamingSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, @@ -51,9 +54,9 @@ public class StreamingSearchCluster extends SearchCluster implements protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } public final String getStorageRouteSpec() { return storageRouteSpec; } - public String getDocTypeName() { - return docTypeName; - } + public String getDocTypeName() { return docTypeName; } + + public DerivedConfiguration derived() { return derivedConfig; } @Override public int getRowBits() { return 0; } @@ -61,10 +64,10 @@ public class StreamingSearchCluster extends SearchCluster implements @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); - docDb.name(schemaConfig.getSchema().getName()); - SummaryConfig.Producer prod = schemaConfig.getSummaries(); + docDb.name(derivedConfig.getSchema().getName()); + SummaryConfig.Producer prod = derivedConfig.getSummaries(); convertSummaryConfig(prod, null, docDb); - addRankProfilesConfig(schemaConfig.getSchema().getName(), docDb); + addRankProfilesConfig(derivedConfig.getSchema().getName(), docDb); builder.documentdb(docDb); } @@ -77,46 +80,58 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.schemaConfig = new DerivedConfiguration(schema, deployState); + this.derivedConfig = new DerivedConfiguration(schema, deployState); } @Override - public DerivedConfiguration getSchemaConfig() { return schemaConfig; } + public void defaultDocumentsConfig() { } @Override - public void defaultDocumentsConfig() { } + public void getConfig(IndexInfoConfig.Builder builder) { + derivedConfig.getIndexInfo().getConfig(builder); + } + + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + derivedConfig.getSchemaInfo().getConfig(builder); + } @Override + public void getConfig(IlscriptsConfig.Builder builder) { + derivedConfig.getIndexingScript().getConfig(builder); + } + public void getConfig(AttributesConfig.Builder builder) { - if (getSchemaConfig() != null) getSchemaConfig().getConfig(builder); + derivedConfig.getConfig(builder); } - + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + derivedConfig.getRankProfileList().getConfig(builder); + } + @Override public void getConfig(VsmsummaryConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getVsmSummary() != null) - getSchemaConfig().getVsmSummary().getConfig(builder); + if (derivedConfig.getVsmSummary() != null) + derivedConfig.getVsmSummary().getConfig(builder); } @Override public void getConfig(VsmfieldsConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getVsmFields() != null) - getSchemaConfig().getVsmFields().getConfig(builder); + if (derivedConfig.getVsmFields() != null) + derivedConfig.getVsmFields().getConfig(builder); } @Override public void getConfig(SummarymapConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getSummaryMap() != null) - getSchemaConfig().getSummaryMap().getConfig(builder); + if (derivedConfig.getSummaryMap() != null) + derivedConfig.getSummaryMap().getConfig(builder); } @Override public void getConfig(SummaryConfig.Builder builder) { - if (getSchemaConfig() != null) - if (getSchemaConfig().getSummaries() != null) - getSchemaConfig().getSummaries().getConfig(builder); + if (derivedConfig.getSummaries() != null) + derivedConfig.getSummaries().getConfig(builder); } private class AttributesProducer extends AbstractConfigProducer<AttributesProducer> implements AttributesConfig.Producer { @@ -127,9 +142,7 @@ public class StreamingSearchCluster extends SearchCluster implements @Override public void getConfig(AttributesConfig.Builder builder) { - if (getSchemaConfig() != null) { - getSchemaConfig().getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); - } + derivedConfig.getConfig(builder, AttributeFields.FieldSet.FAST_ACCESS); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 0d51e4b6e74..81c7b778ed3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -33,17 +33,19 @@ public class MockSearchClusters { } @Override - protected SearchCluster.IndexingMode getIndexingMode() { return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME; } + public void defaultDocumentsConfig() {} @Override - public void getConfig(DocumentdbInfoConfig.Builder builder) { + public void getConfig(AttributesConfig.Builder builder) { } @Override - public void defaultDocumentsConfig() {} + public void getConfig(DocumentdbInfoConfig.Builder builder) { + } @Override - public DerivedConfiguration getSchemaConfig() { return null; } + public void getConfig(RankProfilesConfig.Builder builder) { + } @Override public void getConfig(IndexInfoConfig.Builder builder) { @@ -54,12 +56,11 @@ public class MockSearchClusters { } @Override - public void getConfig(AttributesConfig.Builder builder) { + public void getConfig(SchemaInfoConfig.Builder builder) { } @Override - public void getConfig(RankProfilesConfig.Builder builder) { - } + protected SearchCluster.IndexingMode getIndexingMode() { return streaming ? SearchCluster.IndexingMode.STREAMING : SearchCluster.IndexingMode.REALTIME; } } |