Merge pull request #29824 from vespa-engine/balder/do-not-normalize-streaming-search-terms

No normalization should take place for streaming search. This is hand…

8 files changed, 67 insertions, 11 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
index 1c3c088e1fc..7f874d04f13 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java
@@ -55,7 +55,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer {
     }
 
     DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) {
-        this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build());
+        this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build(), false);
     }
 
     /**
@@ -65,7 +65,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer {
      *               argument is live. Which means that this object will be inconsistent if the given
      *               schema is later modified.
      */
-    public DerivedConfiguration(Schema schema, DeployState deployState) {
+    public DerivedConfiguration(Schema schema, DeployState deployState, boolean isStreaming) {
         try {
             Validator.ensureNotNull("Schema", schema);
             this.schema = schema;
@@ -81,7 +81,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer {
                 juniperrc = new Juniperrc(schema);
                 rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState);
                 indexingScript = new IndexingScript(schema);
-                indexInfo = new IndexInfo(schema);
+                indexInfo = new IndexInfo(schema, isStreaming);
                 schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries);
                 indexSchema = new IndexSchema(schema);
                 importedFields = new ImportedFields(schema);
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java
index 96c64ff1a1c..7532dec5187 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java
@@ -55,12 +55,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
     private static final String CMD_INTEGER = "integer";
     private static final String CMD_STRING = "string";
     private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting";
+    private final boolean isStreaming;
     private final Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
     private final Map<String, String> aliases = new java.util.LinkedHashMap<>();
     private final Map<String, FieldSet> fieldSets;
     private Schema schema;
 
-    public IndexInfo(Schema schema) {
+    public IndexInfo(Schema schema, boolean isStreaming) {
+        this.isStreaming = isStreaming;
         this.fieldSets = schema.fieldSets().userFieldSets();
         addIndexCommand("sddocname", CMD_INDEX);
         addIndexCommand("sddocname", CMD_WORD);
@@ -223,7 +225,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
     }
 
     private boolean normalizeAccents(ImmutableSDField field) {
-        return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING);
+        return !isStreaming && field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING);
     }
 
     private boolean isTypeOrNested(ImmutableSDField field, DataType type) {
@@ -370,7 +372,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
                 anyStemming = true;
                 stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode();
             }
-            if (field.getNormalizing().doRemoveAccents()) {
+            if (normalizeAccents(field)) {
                 anyNormalizing = true;
             }
             if (isTypeOrNested(field, DataType.STRING)) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 81f071ba033..a79bc14db52 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -138,7 +138,7 @@ public class IndexedSearchCluster extends SearchCluster
         for (SchemaInfo spec : schemas().values()) {
             if (spec.fullSchema() instanceof DocumentOnlySchema) continue;
             DocumentDatabase db = new DocumentDatabase(this, spec.fullSchema().getName(),
-                                                       new DerivedConfiguration(spec.fullSchema(), deployState));
+                                                       new DerivedConfiguration(spec.fullSchema(), deployState, false));
             documentDbs.add(db);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
index 3d48432fbe7..f25ca7e4452 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -84,7 +84,7 @@ public class StreamingSearchCluster extends SearchCluster implements
         if ( ! schema.getName().equals(docTypeName))
             throw new IllegalArgumentException("Document type name '" + docTypeName +
                                                "' must be the same as the schema name '" + schema.getName() + "'");
-        this.derivedConfig = new DerivedConfiguration(schema, deployState);
+        this.derivedConfig = new DerivedConfiguration(schema, deployState, true);
     }
 
     @Override
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
index 3bb129a4c32..7686289f11c 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java
@@ -52,7 +52,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase {
                                                                                         .deployLogger(logger)
                                                                                         .rankProfileRegistry(builder.getRankProfileRegistry())
                                                                                         .queryProfiles(builder.getQueryProfileRegistry())
-                                                                                        .build());
+                                                                                        .build(), false);
         return export(dirName, builder, config);
     }
 
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java
new file mode 100644
index 00000000000..09450fa8023
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexInfoTestCase.java
@@ -0,0 +1,55 @@
+package com.yahoo.schema.derived;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder;
+import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
+import com.yahoo.vespa.model.content.utils.DocType;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class IndexInfoTestCase {
+    private static final String F = "f";
+    @Test
+    void testThatIndexingEnablesNormalizing() {
+        var cmds = createIndexCmds(false);
+        assertEquals(8, cmds.size());
+        assertEquals(1, cmds.stream().filter(c -> c.indexname().equals(F) && c.command().equals("normalize")).count());
+    }
+    @Test
+    void testThatStreamingDisablesNormalizing() {
+        var cmds = createIndexCmds(true);
+        assertEquals(7, cmds.size());
+        assertEquals(0, cmds.stream().filter(c -> c.indexname().equals(F) && c.command().equals("normalize")).count());
+    }
+
+    private static List<IndexInfoConfig.Indexinfo.Command> createIndexCmds(boolean isStreaming) {
+        final String SD = "sda";
+        String documentContent = "field " + F + " type string {indexing:index | summary}";
+        var cfg = createIndexInfo(SD, documentContent, isStreaming);
+        assertEquals(SD, cfg.indexinfo(0).name());
+        return cfg.indexinfo(0).command();
+    }
+
+    private static IndexInfoConfig createIndexInfo(String schemaName, String sdContent, boolean isStreaming) {
+        var model = createModel(schemaName, sdContent);
+        var schema = model.getSearchClusters().get(0).schemas().get(schemaName);
+        var indexInfo = new IndexInfo(schema.fullSchema(), isStreaming);
+        IndexInfoConfig.Builder builder = new IndexInfoConfig.Builder();
+        indexInfo.getConfig(builder);
+        return builder.build();
+    }
+
+    private static VespaModel createModel(String schemaName, String sdContent) {
+        var builder = new DeployState.Builder();
+        return new ApplicationPackageBuilder()
+                .addCluster(new ContentClusterBuilder().name("content").docTypes(List.of(DocType.index(schemaName))))
+                .addSchemas(new SchemaBuilder().name(schemaName).content(sdContent).build())
+                .buildCreator().create(builder);
+    }
+}
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
index b219d84f108..2beab3d5ea9 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java
@@ -46,7 +46,7 @@ public class SchemaToDerivedConfigExporter {
                         .deployLogger(logger)
                         .rankProfileRegistry(builder.getRankProfileRegistry())
                         .queryProfiles(builder.getQueryProfileRegistry())
-                        .build());
+                        .build(), false);
         exportConfig(dirPath, derived, builder);
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
index 6defca17bcb..304f3dc426f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.model.content.utils;
 
 import java.util.Arrays;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import static com.yahoo.config.model.test.TestUtil.joinLines;