From 5fb35b0e41b0f62d9b66fd9c4fcee23f82a8400f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 5 Jan 2024 16:32:06 +0100 Subject: No normalization should take place for streaming search. This is handled symmetrically in the backend during query time. --- .../main/java/com/yahoo/schema/derived/DerivedConfiguration.java | 6 +++--- .../src/main/java/com/yahoo/schema/derived/IndexInfo.java | 8 +++++--- .../java/com/yahoo/vespa/model/search/IndexedSearchCluster.java | 2 +- .../java/com/yahoo/vespa/model/search/StreamingSearchCluster.java | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'config-model/src/main/java') diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 1c3c088e1fc..7f874d04f13 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -55,7 +55,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false); } /** @@ -65,7 +65,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { * argument is live. Which means that this object will be inconsistent if the given * schema is later modified. */ - public DerivedConfiguration(Schema schema, DeployState deployState) { + public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) { try { Validator.ensureNotNull("Schema", schema); this.schema = schema; @@ -81,7 +81,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); indexingScript = new IndexingScript(schema); - indexInfo = new IndexInfo(schema); + indexInfo = new IndexInfo(schema, isStreaming); schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 96c64ff1a1c..7532dec5187 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -55,12 +55,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static final String CMD_INTEGER = "integer"; private static final String CMD_STRING = "string"; private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting"; + private final boolean isStreaming; private final Set commands = new java.util.LinkedHashSet<>(); private final Map aliases = new java.util.LinkedHashMap<>(); private final Map fieldSets; private Schema schema; - public IndexInfo(Schema schema) { + public IndexInfo(Schema schema, boolean isStreaming) { + this.isStreaming = isStreaming; this.fieldSets = schema.fieldSets().userFieldSets(); addIndexCommand("sddocname", CMD_INDEX); addIndexCommand("sddocname", CMD_WORD); @@ -223,7 +225,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } private boolean normalizeAccents(ImmutableSDField field) { - return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); + return !isStreaming && field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); } private boolean isTypeOrNested(ImmutableSDField field, DataType type) { @@ -370,7 +372,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { anyStemming = true; stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode(); } - if (field.getNormalizing().doRemoveAccents()) { + if (normalizeAccents(field)) { anyNormalizing = true; } if (isTypeOrNested(field, DataType.STRING)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 81f071ba033..a79bc14db52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -138,7 +138,7 @@ public class IndexedSearchCluster extends SearchCluster for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; DocumentDatabase db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(spec.fullSchema(), deployState)); + new DerivedConfiguration(spec.fullSchema(), deployState, false)); documentDbs.add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 3d48432fbe7..f25ca7e4452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -84,7 +84,7 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.derivedConfig = new DerivedConfiguration(schema, deployState); + this.derivedConfig = new DerivedConfiguration(schema, deployState, true); } @Override -- cgit v1.2.3