Add feature flag for controlling termwise limit default.

9 files changed, 90 insertions, 51 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index d974db73547..0dc08c19b92 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties {
     private boolean useFdispatchByDefault = true;
     private boolean dispatchWithProtobuf = true;
     private boolean useAdaptiveDispatch = false;
+    private double defaultTermwiseLimit = 1.0;
     private Optional<TlsSecrets> tlsSecrets = Optional.empty();
 
 
@@ -62,7 +63,12 @@ public class TestProperties implements ModelContext.Properties {
     @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; }
     @Override public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; }
     @Override public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; }
+    @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
 
+    public TestProperties setDefaultTermwiseLimit(double limit) {
+        defaultTermwiseLimit = limit;
+        return this;
+    }
     public TestProperties setApplicationId(ApplicationId applicationId) {
         this.applicationId = applicationId;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 7a34ff258b7..d3e32868f62 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -501,10 +501,10 @@ public class RankProfile implements Serializable, Cloneable {
                 : numSearchPartitions;
     }
 
-    public double getTermwiseLimit() {
+    public Optional<Double> getTermwiseLimit() {
         return ((termwiseLimit == null) && (getInherited() != null))
                 ?  getInherited().getTermwiseLimit()
-                : (termwiseLimit != null) ? termwiseLimit : 1.0;
+                : (termwiseLimit != null) ? Optional.of(termwiseLimit) : Optional.empty();
     }
     public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index f25dd0be7b2..1e45ea22567 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -3,8 +3,10 @@ package com.yahoo.searchdefinition.derived;
 
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.io.IOUtils;
@@ -50,7 +52,7 @@ public class DerivedConfiguration {
                                 RankProfileRegistry rankProfileRegistry,
                                 QueryProfileRegistry queryProfiles,
                                 ImportedMlModels importedModels) {
-        this(search, new BaseDeployLogger(), rankProfileRegistry, queryProfiles, importedModels);
+        this(search, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, importedModels);
     }
 
     /**
@@ -59,13 +61,14 @@ public class DerivedConfiguration {
      * @param search             The search to derive a configuration from. Derived objects will be snapshots, but this
      *                           argument is live. Which means that this object will be inconsistent when the given
      *                           search definition is later modified.
-     * @param deployLogger       a {@link DeployLogger} for logging when
-     *                           doing operations on this
+     * @param deployLogger       a {@link DeployLogger} for logging when doing operations on this
+     * @param deployProperties   Properties set on deploy.
      * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
      * @param queryProfiles      the query profiles of this application
      */
     public DerivedConfiguration(Search search,
                                 DeployLogger deployLogger,
+                                ModelContext.Properties deployProperties,
                                 RankProfileRegistry rankProfileRegistry,
                                 QueryProfileRegistry queryProfiles,
                                 ImportedMlModels importedModels) {
@@ -80,7 +83,7 @@ public class DerivedConfiguration {
             summaries = new Summaries(search, deployLogger);
             summaryMap = new SummaryMap(search);
             juniperrc = new Juniperrc(search);
-            rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels);
+            rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels, deployProperties);
             indexingScript = new IndexingScript(search);
             indexInfo = new IndexInfo(search);
             indexSchema = new IndexSchema(search);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
index d8f2141d837..f6d4889b55e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.derived;
 
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.RankingConstant;
@@ -45,29 +46,29 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
                            AttributeFields attributeFields,
                            RankProfileRegistry rankProfileRegistry,
                            QueryProfileRegistry queryProfiles,
-                           ImportedMlModels importedModels) {
+                           ImportedMlModels importedModels,
+                           ModelContext.Properties deployProperties) {
         setName(search == null ? "default" : search.getName());
         this.rankingConstants = rankingConstants;
-        deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields);
+        deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties);
     }
 
     private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry,
                                     QueryProfileRegistry queryProfiles,
                                     ImportedMlModels importedModels,
                                     Search search,
-                                    AttributeFields attributeFields) {
+                                    AttributeFields attributeFields,
+                                    ModelContext.Properties deployProperties) {
         if (search != null) { // profiles belonging to a search have a default profile
             RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.get(search, "default"),
-                                                               queryProfiles,
-                                                               importedModels,
-                                                               attributeFields);
+                                                               queryProfiles, importedModels, attributeFields, deployProperties);
             rankProfiles.put(defaultProfile.getName(), defaultProfile);
         }
 
         for (RankProfile rank : rankProfileRegistry.rankProfilesOf(search)) {
             if (search != null && "default".equals(rank.getName())) continue;
 
-            RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, importedModels, attributeFields);
+            RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, importedModels, attributeFields, deployProperties);
             rankProfiles.put(rawRank.getName(), rawRank);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index bad46955dd9..e2955d4b3ff 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -5,6 +5,8 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import com.google.common.collect.ImmutableList;
 import com.yahoo.collections.Pair;
 import com.yahoo.compress.Compressor;
+import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.document.RankType;
 import com.yahoo.searchdefinition.RankProfile;
@@ -16,7 +18,7 @@ import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.vespa.config.search.RankProfilesConfig;
 
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -50,20 +52,28 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
     /**
      * Creates a raw rank profile from the given rank profile
      */
+    public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) {
+        this.name = rankProfile.getName();
+        compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, deployProperties).derive());
+    }
+
+    /**
+     * Only for testing
+     */
     public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) {
         this.name = rankProfile.getName();
-        compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive());
+        compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()).derive());
     }
     
     private Compressor.Compression compress(List<Pair<String, String>> properties) {
         StringBuilder b = new StringBuilder();
         for (Pair<String, String> property : properties)
             b.append(property.getFirst()).append(keyEndMarker).append(property.getSecond()).append(valueEndMarker);
-        return compressor.compress(b.toString().getBytes(Charset.forName("utf8")));
+        return compressor.compress(b.toString().getBytes(StandardCharsets.UTF_8));
     }
 
     private List<Pair<String, String>> decompress(Compressor.Compression compression) {
-        String propertiesString = new String(compressor.decompress(compression), Charset.forName("utf8"));
+        String propertiesString = new String(compressor.decompress(compression), StandardCharsets.UTF_8);
         if (propertiesString.isEmpty()) return ImmutableList.of();
 
         ImmutableList.Builder<Pair<String, String>> properties = new ImmutableList.Builder<>();
@@ -148,11 +158,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
         /**
          * Creates a raw rank profile from the given rank profile
          */
-        public Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) {
+        Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels,
+                       AttributeFields attributeFields, ModelContext.Properties deployProperties)
+        {
             RankProfile compiled = rankProfile.compile(queryProfiles, importedModels);
             attributeTypes = compiled.getAttributeTypes();
             queryFeatureTypes = compiled.getQueryFeatureTypes();
-            deriveRankingFeatures(compiled);
+            deriveRankingFeatures(compiled, deployProperties);
             deriveRankTypeSetting(compiled, attributeFields);
             deriveFilterFields(compiled);
             deriveWeightProperties(compiled);
@@ -162,7 +174,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             filterFields.addAll(rp.allFilterFields());
         }
 
-        public void deriveRankingFeatures(RankProfile rankProfile) {
+        private void deriveRankingFeatures(RankProfile rankProfile, ModelContext.Properties deployProperties) {
             firstPhaseRanking = rankProfile.getFirstPhaseRanking();
             secondPhaseRanking = rankProfile.getSecondPhaseRanking();
             summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures());
@@ -172,7 +184,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             numThreadsPerSearch = rankProfile.getNumThreadsPerSearch();
             minHitsPerThread = rankProfile.getMinHitsPerThread();
             numSearchPartitions = rankProfile.getNumSearchPartitions();
-            termwiseLimit = rankProfile.getTermwiseLimit();
+            termwiseLimit = rankProfile.getTermwiseLimit().orElse(deployProperties.defaultTermwiseLimit());
             keepRankCount = rankProfile.getKeepRankCount();
             rankScoreDropLimit = rankProfile.getRankScoreDropLimit();
             ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures();
@@ -263,7 +275,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             }
         }
 
-        public void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) {
+        private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) {
             if (isDefaultSetting) return;
 
             NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType);
@@ -290,7 +302,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             return setting != null && setting.getValue().equals(RankType.DEFAULT);
         }
 
-        public FieldRankSettings deriveFieldRankSettings(String fieldName) {
+        private FieldRankSettings deriveFieldRankSettings(String fieldName) {
             FieldRankSettings settings = fieldRankSettings.get(fieldName);
             if (settings == null) {
                 settings = new FieldRankSettings(fieldName);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index f36cc3b195c..31746c39f80 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -164,11 +164,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
         createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(),
                                  deployState.rankProfileRegistry(), deployState.getQueryProfiles());
         this.rankProfileList = new RankProfileList(null, // null search -> global
-                                                   rankingConstants,
-                                                   AttributeFields.empty,
+                                                   rankingConstants, AttributeFields.empty,
                                                    deployState.rankProfileRegistry(),
                                                    deployState.getQueryProfiles().getRegistry(),
-                                                   deployState.getImportedModels());
+                                                   deployState.getImportedModels(),
+                                                   deployState.getProperties());
 
 
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 1d804ba3b27..c12ef3cf30f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -304,6 +304,7 @@ public class IndexedSearchCluster extends SearchCluster
             if ( ! (search instanceof DocumentOnlySearch)) {
                 DocumentDatabase db = new DocumentDatabase(this, search.getName(),
                                                            new DerivedConfiguration(search, deployState.getDeployLogger(),
+                                                                                    deployState.getProperties(),
                                                                                     deployState.rankProfileRegistry(),
                                                                                     deployState.getQueryProfiles().getRegistry(),
                                                                                     deployState.getImportedModels()));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
index 832f5bf7d6b..d668adea116 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -103,6 +103,7 @@ public class StreamingSearchCluster extends SearchCluster implements
             throw new IllegalStateException("Mismatch between document type name (" + docTypeName + ") and name of search definition (" + localSearch.getName() + ")");
         }
         this.sdConfig = new DerivedConfiguration(localSearch, deployState.getDeployLogger(),
+                                                 deployState.getProperties(),
                                                  deployState.rankProfileRegistry(),
                                                  deployState.getQueryProfiles().getRegistry(),
                                                  deployState.getImportedModels());
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
index 9f59801412c..5ba508e3ef3 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
@@ -3,7 +3,9 @@ package com.yahoo.searchdefinition;
 
 import com.yahoo.collections.Pair;
 import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.document.DataType;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.types.FieldDescription;
@@ -58,42 +60,55 @@ public class RankProfileTestCase extends SearchDefinitionTestCase {
         assertEquals(RankType.DEFAULT, setting.getValue());
     }
 
+    private String createSD(Double termwiseLimit) {
+        return "search test {\n" +
+                "    document test { \n" +
+                "        field a type string { \n" +
+                "            indexing: index \n" +
+                "        }\n" +
+                "    }\n" +
+                "    \n" +
+                "    rank-profile parent {\n" +
+                (termwiseLimit != null ? ("        termwise-limit:" + termwiseLimit + "\n") : "") +
+                "        num-threads-per-search:8\n" +
+                "        min-hits-per-thread:70\n" +
+                "        num-search-partitions:1200\n" +
+                "    }\n" +
+                "    rank-profile child inherits parent { }\n" +
+                "}\n";
+    }
+
     @Test
-    public void testTermwiseLimitAndSomeMoreIncludingInheritance() throws ParseException {
+    public void testTermwiseLimitWithDeployOverride() throws ParseException {
+        verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties(), createSD(null), null);
+        verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties(), createSD(0.78), 0.78);
+        verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties().setDefaultTermwiseLimit(0.09), createSD(null), 0.09);
+        verifyTermwiseLimitAndSomeMoreIncludingInheritance(new TestProperties().setDefaultTermwiseLimit(0.09), createSD(0.37), 0.37);
+    }
+
+    private void verifyTermwiseLimitAndSomeMoreIncludingInheritance(ModelContext.Properties deployProperties, String sd, Double termwiseLimit) throws ParseException {
         RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
         SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
-        builder.importString(
-                "search test {\n" +
-                        "    document test { \n" +
-                        "        field a type string { \n" +
-                        "            indexing: index \n" +
-                        "        }\n" +
-                        "    }\n" +
-                        "    \n" +
-                        "    rank-profile parent {\n" +
-                        "        termwise-limit:0.78\n" +
-                        "        num-threads-per-search:8\n" +
-                        "        min-hits-per-thread:70\n" +
-                        "        num-search-partitions:1200\n" +
-                        "    }\n" +
-                        "    rank-profile child inherits parent { }" +
-                        "\n" +
-                        "}\n");
+        builder.importString(sd);
         builder.build();
         Search search = builder.getSearch();
         AttributeFields attributeFields = new AttributeFields(search);
-        verifyRankProfile(rankProfileRegistry.get(search, "parent"), attributeFields);
-        verifyRankProfile(rankProfileRegistry.get(search, "child"), attributeFields);
+        verifyRankProfile(rankProfileRegistry.get(search, "parent"), attributeFields, deployProperties, termwiseLimit);
+        verifyRankProfile(rankProfileRegistry.get(search, "child"), attributeFields, deployProperties, termwiseLimit);
     }
 
-    private void verifyRankProfile(RankProfile rankProfile, AttributeFields attributeFields) {
-        assertEquals(0.78, rankProfile.getTermwiseLimit(), 0.000001);
+    private void verifyRankProfile(RankProfile rankProfile, AttributeFields attributeFields, ModelContext.Properties deployProperties,
+                                   Double expectedTermwiseLimit) {
         assertEquals(8, rankProfile.getNumThreadsPerSearch());
         assertEquals(70, rankProfile.getMinHitsPerThread());
         assertEquals(1200, rankProfile.getNumSearchPartitions());
-        RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, new QueryProfileRegistry(), new ImportedMlModels(), attributeFields);
-        assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent());
-        assertEquals("0.78", findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").get());
+        RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, new QueryProfileRegistry(), new ImportedMlModels(), attributeFields, deployProperties);
+        if (expectedTermwiseLimit != null) {
+            assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent());
+            assertEquals(String.valueOf(expectedTermwiseLimit), findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").get());
+        } else {
+            assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent());
+        }
         assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").isPresent());
         assertEquals("8", findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").get());
         assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.minhitsperthread").isPresent());