Propagate phrase-segmenting to fieldsets

6 files changed, 52 insertions, 15 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 0b4562ecd5c..2e4cdd706f7 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -42,7 +42,7 @@ public class TestProperties implements ModelContext.Properties {
     private double defaultTermwiseLimit = 1.0;
     private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty();
     private boolean useNewAthenzFilter = false;
-
+    private boolean usePhraseSegmenting = false;
 
     @Override public boolean multitenant() { return multitenant; }
     @Override public ApplicationId applicationId() { return applicationId; }
@@ -63,6 +63,7 @@ public class TestProperties implements ModelContext.Properties {
     @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
     @Override public boolean useBucketSpaceMetric() { return true; }
     @Override public boolean useNewAthenzFilter() { return useNewAthenzFilter; }
+    @Override public boolean usePhraseSegmenting() { return usePhraseSegmenting; }
 
     public TestProperties setDefaultTermwiseLimit(double limit) {
         defaultTermwiseLimit = limit;
@@ -114,6 +115,11 @@ public class TestProperties implements ModelContext.Properties {
         return this;
     }
 
+    public TestProperties setUsePhraseSegmenting(boolean phraseSegmenting) {
+        this.usePhraseSegmenting = phraseSegmenting;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
index f30a150aeee..9ae72badf1c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.documentmodel.SummaryField;
 import com.yahoo.search.config.IndexInfoConfig;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -301,6 +302,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         boolean anyLowerCasing = false;
         boolean anyStemming = false;
         boolean anyNormalizing = false;
+        String phraseSegmentingCommand = null;
         String stemmingCommand = null;
         Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching
         // First a pass over the fields to read some params to decide field settings implicitly:
@@ -321,8 +323,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             if (field.getNormalizing().doRemoveAccents()) {
                 anyNormalizing = true;
             }
-            if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType)
+            if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) {
                 fieldSetMatching = field.getMatching();
+            }
+            Optional<String> explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst();
+            if (explicitPhraseSegmentingCommand.isPresent()) {
+                phraseSegmentingCommand = explicitPhraseSegmentingCommand.get();
+            }
         }
         if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) {
             // We have both attributes and indexes and no explicit match setting ->
@@ -365,6 +372,11 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
                             new IndexInfoConfig.Indexinfo.Command.Builder()
                                 .indexname(fieldSet.getName())
                                 .command(CMD_NORMALIZE));
+                if (phraseSegmentingCommand != null)
+                    iiB.command(
+                            new IndexInfoConfig.Indexinfo.Command.Builder()
+                                    .indexname(fieldSet.getName())
+                                    .command(phraseSegmentingCommand));
             }
         } else {
             // Assume only attribute fields
@@ -400,13 +412,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) {
                 
             }
-            
         }
 
-        if (phraseSegmenting) {
-            iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder()
-                                .indexname(fieldSet.getName())
-                                .command(CMD_PHRASE_SEGMENTING));
+        if (phraseSegmentingCommand == null
+            && fieldSet.queryCommands().stream().noneMatch(c -> c.startsWith(CMD_PHRASE_SEGMENTING))) { // use default
+            if (phraseSegmenting)
+                iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder()
+                                    .indexname(fieldSet.getName())
+                                    .command(CMD_PHRASE_SEGMENTING));
         }
     }
 
diff --git a/config-model/src/test/derived/fieldset2/index-info.cfg b/config-model/src/test/derived/fieldset2/index-info.cfg
index 7c3c1c448db..56cc53b4628 100644
--- a/config-model/src/test/derived/fieldset2/index-info.cfg
+++ b/config-model/src/test/derived/fieldset2/index-info.cfg
@@ -13,6 +13,8 @@ indexinfo[].command[].indexname "field1"
 indexinfo[].command[].command "normalize"
 indexinfo[].command[].indexname "field1"
 indexinfo[].command[].command "plain-tokens"
+indexinfo[].command[].indexname "field1"
+indexinfo[].command[].command "phrase-segmenting"
 indexinfo[].command[].indexname "field2"
 indexinfo[].command[].command "index"
 indexinfo[].command[].indexname "field2"
@@ -23,6 +25,8 @@ indexinfo[].command[].indexname "field2"
 indexinfo[].command[].command "normalize"
 indexinfo[].command[].indexname "field2"
 indexinfo[].command[].command "plain-tokens"
+indexinfo[].command[].indexname "field2"
+indexinfo[].command[].command "phrase-segmenting"
 indexinfo[].command[].indexname "default"
 indexinfo[].command[].command "phrase-segmenting false"
 indexinfo[].command[].indexname "default"
diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg
index 65818e088f5..388b212689a 100644
--- a/config-model/src/test/derived/indexschema/index-info.cfg
+++ b/config-model/src/test/derived/indexschema/index-info.cfg
@@ -334,6 +334,8 @@ indexinfo[].command[].command "stem:BEST"
 indexinfo[].command[].indexname "gram"
 indexinfo[].command[].command "normalize"
 indexinfo[].command[].indexname "gram"
+indexinfo[].command[].command "phrase-segmenting false"
+indexinfo[].command[].indexname "gram"
 indexinfo[].command[].command "ngram 2"
 indexinfo[].command[].indexname "nostem1"
 indexinfo[].command[].command "lowercase"
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
index 8ea53172200..7e0eb01f611 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.searchdefinition.Search;
@@ -10,8 +12,6 @@ import com.yahoo.searchdefinition.parser.ParseException;
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import com.yahoo.vespa.configmodel.producers.DocumentManager;
 import com.yahoo.vespa.configmodel.producers.DocumentTypes;
-import com.yahoo.vespa.model.container.search.QueryProfiles;
-import com.yahoo.vespa.model.test.utils.DeployLoggerStub;
 
 import java.io.File;
 import java.io.IOException;
@@ -26,17 +26,22 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
     private static final String tempDir = "temp/";
     private static final String searchDefRoot = "src/test/derived/";
 
-    private DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException {
+    private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
         File toDir = new File(tempDir + dirName);
         toDir.mkdirs();
         deleteContent(toDir);
 
         SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/");
-        return derive(dirName, searchDefinitionName, builder);
+        return derive(dirName, searchDefinitionName, properties, builder);
     }
 
-    private DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException {
+    private DerivedConfiguration derive(String dirName,
+                                        String searchDefinitionName,
+                                        TestProperties properties,
+                                        SearchBuilder builder) throws IOException {
         DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName),
+                                                               new BaseDeployLogger(),
+                                                               properties,
                                                                builder.getRankProfileRegistry(),
                                                                builder.getQueryProfileRegistry(),
                                                                new ImportedMlModels());
@@ -79,7 +84,11 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
     }
 
     protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException {
-        DerivedConfiguration derived = derive(dirName, searchDefinitionName);
+        return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties());
+    }
+
+    protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
+        DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties);
         assertCorrectConfigFiles(dirName);
         return derived;
     }
@@ -90,7 +99,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
      */
     protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException {
         builder.build();
-        DerivedConfiguration derived = derive(dirName, null, builder);
+        DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder);
         assertCorrectConfigFiles(dirName);
         return derived;
     }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index e785792839d..08d915c35ca 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.searchdefinition.SearchBuilder;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
@@ -121,7 +122,9 @@ public class ExportingTestCase extends AbstractExportingTestCase {
 
     @Test
     public void testFieldSet2() throws IOException, ParseException {
-        assertCorrectDeriving("fieldset2");
+        TestProperties properties = new TestProperties();
+        properties.setUsePhraseSegmenting(true);
+        assertCorrectDeriving("fieldset2", null, properties);
     }
 
     @Test