diff options
Diffstat (limited to 'config-model/src')
6 files changed, 184 insertions, 68 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index 2d4b4824310..c23d87e9eba 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -57,6 +57,7 @@ public class Processing { AdjustSummaryTransforms::new, SummaryNamesFieldCollisions::new, SummaryFieldsMustHaveValidSource::new, + TokensTransformValidator::new, MatchedElementsOnlyResolver::new, MakeDefaultSummaryTheSuperSet::new, Bolding::new, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java new file mode 100644 index 00000000000..7988a0b9ceb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/TokensTransformValidator.java @@ -0,0 +1,50 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/* + * Check that summary fields with summary transform 'tokens' have a source field with a data type that is one of + * string, array<string> or weightedset<string>. + */ +public class TokensTransformValidator extends Processor { + public TokensTransformValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if (!validate || documentsOnly) { + return; + } + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + if (summaryField.getTransform().isTokens()) { + var source = summaryField.getSingleSource(); + if (source != null) { + var field = schema.getField(source); + if (field != null) { + var type = field.getDataType(); + var innerType = type.getPrimitiveType(); + if (innerType != DataType.STRING) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', document-summary '" + summary.getName() + + "', summary field '" + summaryField.getName() + + "', source field '" + field.getName() + + "', source field type '" + type.getName() + + "': transform '" + SummaryTransform.TOKENS.getName() + + "' is only allowed for fields of type" + + " string, array<string> or weightedset<string>"); + } + } + } + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index 50be01db04b..58f47680f9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -69,6 +69,8 @@ public enum SummaryTransform { return this==DYNAMICBOLDED || this==DYNAMICTEASER; } + public boolean isTokens() { return this == TOKENS; } + /** Returns whether this transform always gets its value by accessing memory only */ public boolean isInMemory() { return switch (this) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index ca2bc40209d..d86d117f1d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -5,24 +5,24 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.ReconfigurableDispatcher; +import com.yahoo.search.handler.observability.SearchStatusExtension; import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.ranking.RankProfilesEvaluatorFactory; -import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; -import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.StreamingSearchCluster; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.handler.observability.SearchStatusExtension; import java.util.Collection; import java.util.LinkedList; @@ -52,7 +52,6 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private final List<SearchCluster> searchClusters = new LinkedList<>(); private final Collection<String> schemasWithGlobalPhase; private final boolean globalPhase; - private final boolean useReconfigurableDispatcher; private QueryProfiles queryProfiles; private SemanticRules semanticRules; @@ -62,7 +61,6 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); this.globalPhase = deployState.featureFlags().enableGlobalPhase(); - this.useReconfigurableDispatcher = deployState.featureFlags().useReconfigurableDispatcher(); this.schemasWithGlobalPhase = getSchemasWithGlobalPhase(deployState); this.app = deployState.getApplicationPackage(); this.owningCluster = cluster; @@ -88,9 +86,12 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> /** Adds a Dispatcher component to the owning container cluster for each search cluster */ private void initializeDispatchers(Collection<SearchCluster> searchClusters) { - Class<? extends Dispatcher> dispatcherClass = useReconfigurableDispatcher ? ReconfigurableDispatcher.class : Dispatcher.class; for (SearchCluster searchCluster : searchClusters) { if (searchCluster instanceof IndexedSearchCluster indexed) { + // For local testing, using Application, there is no cloud config, and we need to use the static dispatcher. + Class<? extends Dispatcher> dispatcherClass = System.getProperty("vespa.local", "false").equals("true") + ? Dispatcher.class + : ReconfigurableDispatcher.class; var dispatcher = new DispatcherComponent(indexed, dispatcherClass); owningCluster.addComponent(dispatcher); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java new file mode 100644 index 00000000000..6ca62321617 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java @@ -0,0 +1,59 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.Schema; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import org.junit.jupiter.api.Test; + +import static com.yahoo.config.model.test.TestUtil.joinLines; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +public class TokensTransformValidatorTest { + private void buildSchema(String fieldType) throws ParseException { + String sd = joinLines( + "search test {", + " document test {", + " field f type " + fieldType + " {", + " indexing: summary", + " summary: tokens", + " }", + " }", + "}" + ); + Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); + } + + void buildSchemaShouldFail(String fieldType, String expFail) throws ParseException { + try { + buildSchema(fieldType); + fail("expected IllegalArgumentException with message '" + expFail + "'"); + } catch (IllegalArgumentException e) { + assertEquals(expFail, e.getMessage()); + } + } + + @Test + void testTokensTransformWithPlainString() throws ParseException { + buildSchema("string"); + } + + @Test + void testTokensTransformWithArrayOfString() throws ParseException { + buildSchema("array<string>"); + } + + @Test + void testTokensTransformWithWeightedSetOfString() throws ParseException { + buildSchema("weightedset<string>"); + } + + @Test + void testTokensTransformWithWeightedSetOfInteger() throws ParseException { + buildSchemaShouldFail("weightedset<int>", "For schema 'test', document-summary 'default'" + + ", summary field 'f', source field 'f', source field type 'WeightedSet<int>'" + + ": transform 'tokens' is only allowed for fields of type string, array<string> or weightedset<string>"); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index d63b4586082..c8d84f24581 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -11,6 +11,7 @@ import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; +import com.yahoo.search.dispatch.ReconfigurableDispatcher; import com.yahoo.vespa.config.search.DispatchNodesConfig; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; @@ -62,68 +63,70 @@ public class SchemaClusterTest { @Test void search_model_is_connected_to_container_clusters_two_content_clusters() { - String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" + - "<hosts>" + - " <host name='node0host'>" + - " <alias>node0</alias>" + - " </host>" + - " <host name='node1host'>" + - " <alias>node1</alias>" + - " </host>" + - " <host name='node2host'>" + - " <alias>node2</alias>" + - " </host>" + - "</hosts>"; + String vespaHosts = """ + <?xml version='1.0' encoding='utf-8' ?> + <hosts> + <host name='node0host'> + <alias>node0</alias> + </host> + <host name='node1host'> + <alias>node1</alias> + </host> + <host name='node2host'> + <alias>node2</alias> + </host> + </hosts> + """; String services = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<services version=\"1.0\">" + - " <admin version='2.0'>" + - " <adminserver hostalias='node0' />" + - " </admin>\n" + - " <container version='1.0' id='j1'>\n" + - " <search>" + - " <chain id='s1Chain'>" + - " <searcher id='S1ClusterSearcher'/>" + - " </chain>" + - " <provider cluster='normal' id='normal' type='local'/>\n" + - " </search>" + - " <nodes>" + - " <node hostalias=\"node0\" />" + - " </nodes>" + - " </container>" + - - " <container version='1.0' id='j2'>" + - " <search>" + - " <chain id='s2Chain'>" + - " <searcher id='S2ClusterSearcher'/>" + - " </chain>" + - " <provider cluster='xbulk' id='xbulk' type='local'/>" + - " </search>" + - " <nodes>" + - " <node hostalias=\"node2\" />" + - " </nodes>" + - " </container>" + - - " <content id='xbulk' version=\"1.0\">" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document mode='index' type=\"music\" />" + - " </documents>" + - " <nodes>" + - " <node hostalias=\"node0\" distribution-key=\"0\" />" + - " </nodes>" + - " </content>" + - " <content id=\"normal\" version='1.0'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document mode='index' type=\"music\" />" + - " </documents>" + - " <nodes>" + - " <node hostalias=\"node2\" distribution-key=\"0\" />" + - " </nodes>" + - " </content>" + - "</services>"; + """ + <?xml version="1.0" encoding="utf-8" ?> + <services version="1.0"> + <admin version='2.0'> + <adminserver hostalias='node0' /> + </admin> + <container version='1.0' id='j1'> + <search> + <chain id='s1Chain'> + <searcher id='S1ClusterSearcher'/> + </chain> + <provider cluster='normal' id='normal' type='local'/> + </search> + <nodes> + <node hostalias="node0" /> + </nodes> + </container> + <container version='1.0' id='j2'> + <search> + <chain id='s2Chain'> + <searcher id='S2ClusterSearcher'/> + </chain> + <provider cluster='xbulk' id='xbulk' type='local'/> + </search> + <nodes> + <node hostalias="node2" /> + </nodes> + </container> + <content id='xbulk' version="1.0"> + <redundancy>2</redundancy> + <documents> + <document mode='index' type="music" /> + </documents> + <nodes> + <node hostalias="node0" distribution-key="0" /> + </nodes> + </content> + <content id="normal" version='1.0'> + <redundancy>2</redundancy> + <documents> + <document mode='index' type="music" /> + </documents> + <nodes> + <node hostalias="node2" distribution-key="0" /> + </nodes> + </content> + </services> + """; VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSchemas("music")).create(); @@ -169,7 +172,7 @@ public class SchemaClusterTest { Component<?,?> dispatcher = (Component<?, ?>)containerCluster.getComponentsMap().get(new ComponentId("dispatcher." + cluster)); assertNotNull(dispatcher); assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue()); - assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue()); + assertEquals(ReconfigurableDispatcher.class.getName(), dispatcher.getClassId().stringValue()); assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId()); DispatchNodesConfig.Builder dispatchConfigBuilder = new DispatchNodesConfig.Builder(); model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId()); |