diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-01-06 21:39:54 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-06 21:39:54 +0100 |
commit | f75e3d22696e10270d6c9637df4daa52b340e583 (patch) | |
tree | 66f7942ca5c7d3a9a564f0f7703de31f7aefca6c /config-model/src | |
parent | 936443d273148db908da50233e54aec74603cbd8 (diff) | |
parent | 64bd1394e806bb5857c872468282c7b4023b33ce (diff) |
Merge pull request #29824 from vespa-engine/balder/do-not-normalize-streaming-search-terms
No normalization should take place for streaming search. This is hand…
Diffstat (limited to 'config-model/src')
8 files changed, 67 insertions, 11 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 1c3c088e1fc..7f874d04f13 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -55,7 +55,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false); } /** @@ -65,7 +65,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { * argument is live. Which means that this object will be inconsistent if the given * schema is later modified. */ - public DerivedConfiguration(Schema schema, DeployState deployState) { + public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -81,7 +81,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); indexingScript = new IndexingScript(schema); - indexInfo = new IndexInfo(schema); + indexInfo = new IndexInfo(schema, isStreaming); schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 96c64ff1a1c..7532dec5187 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -55,12 +55,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static final String CMD_INTEGER = "integer"; private static final String CMD_STRING = "string"; private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; + private final boolean isStreaming; private final Set<IndexCommand> commands = new java.util.LinkedHashSet<>(); private final Map<String, String> aliases = new java.util.LinkedHashMap<>(); private final Map<String, FieldSet> fieldSets; private Schema schema; - public IndexInfo(Schema schema) { + public IndexInfo(Schema schema, boolean isStreaming) { + this.isStreaming = isStreaming; this.fieldSets = schema.fieldSets().userFieldSets(); addIndexCommand("sddocname", CMD_INDEX); addIndexCommand("sddocname", CMD_WORD); @@ -223,7 +225,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } private boolean normalizeAccents(ImmutableSDField field) { - return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); + return !isStreaming && field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); } private boolean isTypeOrNested(ImmutableSDField field, DataType type) { @@ -370,7 +372,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { anyStemming = true; stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode(); } - if (field.getNormalizing().doRemoveAccents()) { + if (normalizeAccents(field)) { anyNormalizing = true; } if (isTypeOrNested(field, DataType.STRING)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 81f071ba033..a79bc14db52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -138,7 +138,7 @@ public class IndexedSearchCluster extends SearchCluster for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; DocumentDatabase db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(spec.fullSchema(), deployState)); + new DerivedConfiguration(spec.fullSchema(), deployState, false)); documentDbs.add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 3d48432fbe7..f25ca7e4452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -84,7 +84,7 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.derivedConfig = new DerivedConfiguration(schema, deployState); + this.derivedConfig = new DerivedConfiguration(schema, deployState, true); } @Override diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java index 3bb129a4c32..7686289f11c 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java @@ -52,7 +52,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { .deployLogger(logger) .rankProfileRegistry(builder.getRankProfileRegistry()) .queryProfiles(builder.getQueryProfileRegistry()) - .build()); + .build(), false); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java new file mode 100644 index 00000000000..09450fa8023 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java @@ -0,0 +1,55 @@ +package com.yahoo.schema.derived; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder; +import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.DocType; +import com.yahoo.vespa.model.content.utils.SchemaBuilder; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class IndexInfoTestCase { + private static final String F = "f"; + @Test + void testThatIndexingEnablesNormalizing() { + var cmds = createIndexCmds(false); + assertEquals(8, cmds.size()); + assertEquals(1, cmds.stream().filter(c -> c.indexname().equals(F) && c.command().equals("normalize")).count()); + } + @Test + void testThatStreamingDisablesNormalizing() { + var cmds = createIndexCmds(true); + assertEquals(7, cmds.size()); + assertEquals(0, cmds.stream().filter(c -> c.indexname().equals(F) && c.command().equals("normalize")).count()); + } + + private static List<IndexInfoConfig.Indexinfo.Command> createIndexCmds(boolean isStreaming) { + final String SD = "sda"; + String documentContent = "field " + F + " type string {indexing:index | summary}"; + var cfg = createIndexInfo(SD, documentContent, isStreaming); + assertEquals(SD, cfg.indexinfo(0).name()); + return cfg.indexinfo(0).command(); + } + + private static IndexInfoConfig createIndexInfo(String schemaName, String sdContent, boolean isStreaming) { + var model = createModel(schemaName, sdContent); + var schema = model.getSearchClusters().get(0).schemas().get(schemaName); + var indexInfo = new IndexInfo(schema.fullSchema(), isStreaming); + IndexInfoConfig.Builder builder = new IndexInfoConfig.Builder(); + indexInfo.getConfig(builder); + return builder.build(); + } + + private static VespaModel createModel(String schemaName, String sdContent) { + var builder = new DeployState.Builder(); + return new ApplicationPackageBuilder() + .addCluster(new ContentClusterBuilder().name("content").docTypes(List.of(DocType.index(schemaName)))) + .addSchemas(new SchemaBuilder().name(schemaName).content(sdContent).build()) + .buildCreator().create(builder); + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java index b219d84f108..2beab3d5ea9 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java @@ -46,7 +46,7 @@ public class SchemaToDerivedConfigExporter { .deployLogger(logger) .rankProfileRegistry(builder.getRankProfileRegistry()) .queryProfiles(builder.getQueryProfileRegistry()) - .build()); + .build(), false); exportConfig(dirPath, derived, builder); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java index 6defca17bcb..304f3dc426f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.content.utils; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; |