Merge pull request #9898 from vespa-engine/geirst/flag-for-bm25-posting-lists

Rename flag to turn on new posting list format with interleaved featu…

8 files changed, 29 insertions, 30 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
index d7e9e0da081..0ea3f5c24a3 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
@@ -56,8 +56,8 @@ public class Index implements Cloneable, Serializable {
     /** The boolean index definition, if set */
     private BooleanIndexDefinition boolIndex;
 
-    // TODO: Remove when experimental posting list format is made default
-    private boolean experimentalPostingListFormat = false;
+    /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */
+    private boolean interleavedFeatures = false;
 
     public Index(String name) {
         this(name, false);
@@ -184,12 +184,12 @@ public class Index implements Cloneable, Serializable {
         boolIndex = def;
     }
 
-    public void setExperimentalPostingListFormat(boolean value) {
-        experimentalPostingListFormat = value;
+    public void setInterleavedFeatures(boolean value) {
+        interleavedFeatures = value;
     }
 
-    public boolean useExperimentalPostingListFormat() {
-        return experimentalPostingListFormat;
+    public boolean useInterleavedFeatures() {
+        return interleavedFeatures;
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
index 3b62807ce73..60b8ee78c7b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
@@ -114,7 +114,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
                 .prefix(f.hasPrefix())
                 .phrases(f.hasPhrases())
                 .positions(f.hasPositions())
-                .interleavedfeatures(f.useExperimentalPostingListFormat());
+                .interleavedfeatures(f.useInterleavedFeatures());
             if (!f.getCollectionType().equals("SINGLE")) {
                 ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType()));
             }
@@ -175,8 +175,8 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
         private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility?
         private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility?
         private BooleanIndexDefinition boolIndex = null;
-        // TODO: Remove when experimental posting list format is made default
-        private boolean experimentalPostingListFormat = false;
+        // Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream.
+        private boolean interleavedFeatures = false;
 
         public IndexField(String name, Index.Type type, DataType sdFieldType) {
             this.name = name;
@@ -186,7 +186,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
         public void setIndexSettings(com.yahoo.searchdefinition.Index index) {
             if (type.equals(Index.Type.TEXT)) {
                 prefix = index.isPrefix();
-                experimentalPostingListFormat = index.useExperimentalPostingListFormat();
+                interleavedFeatures = index.useInterleavedFeatures();
             }
             sdType = index.getType();
             boolIndex = index.getBooleanIndexDefiniton();
@@ -209,7 +209,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
         public boolean hasPrefix() { return prefix; }
         public boolean hasPhrases() { return phrases; }
         public boolean hasPositions() { return positions; }
-        public boolean useExperimentalPostingListFormat() { return experimentalPostingListFormat; }
+        public boolean useInterleavedFeatures() { return interleavedFeatures; }
 
         public BooleanIndexDefinition getBooleanIndexDefinition() {
             return boolIndex;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
index 459bb247e5f..39f543c7db3 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
@@ -29,8 +29,7 @@ public class IndexOperation implements FieldOperation {
     private OptionalLong lowerBound = OptionalLong.empty();
     private OptionalLong upperBound = OptionalLong.empty();
     private OptionalDouble densePostingListThreshold = OptionalDouble.empty();
-    // TODO: Remove when experimental posting list format is made default
-    private Optional<Boolean> experimentalPostingListFormat = Optional.empty();
+    private Optional<Boolean> enableBm25 = Optional.empty();
 
     public String getIndexName() {
         return indexName;
@@ -89,8 +88,8 @@ public class IndexOperation implements FieldOperation {
             index.setBooleanIndexDefiniton(
                     new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold));
         }
-        if (experimentalPostingListFormat.isPresent()) {
-            index.setExperimentalPostingListFormat(experimentalPostingListFormat.get());
+        if (enableBm25.isPresent()) {
+            index.setInterleavedFeatures(enableBm25.get());
         }
     }
 
@@ -117,8 +116,8 @@ public class IndexOperation implements FieldOperation {
     public void setDensePostingListThreshold(double densePostingListThreshold) {
         this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold);
     }
-    public void setExperimentalPostingListFormat(boolean value) {
-        experimentalPostingListFormat = Optional.of(value);
+    public void setEnableBm25(boolean value) {
+        enableBm25 = Optional.of(value);
     }
 
 }
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 571ad452b01..6dde12f0fac 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -331,7 +331,7 @@ TOKEN :
 | < LOWERBOUND: "lower-bound" >
 | < UPPERBOUND: "upper-bound" >
 | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" >
-| < EXPERIMENTALPOSTINGLISTFORMAT: "experimental-posting-list-format" >
+| < ENABLE_BM25: "enable-bm25" >
 | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? >
 | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
 | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? >
@@ -1782,7 +1782,7 @@ Object indexBody(IndexOperation index) :
       | <LOWERBOUND> <COLON> num = consumeLong()                       { index.setLowerBound(num); }
       | <UPPERBOUND> <COLON> num = consumeLong()                       { index.setUpperBound(num); }
       | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); }
-      | <EXPERIMENTALPOSTINGLISTFORMAT>                                { index.setExperimentalPostingListFormat(true); }
+      | <ENABLE_BM25>                                                  { index.setEnableBm25(true); }
     )
     { return null; }
 }
diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg
index 46c2c3fc307..a83ec45c5e9 100644
--- a/config-model/src/test/derived/indexschema/index-info.cfg
+++ b/config-model/src/test/derived/indexschema/index-info.cfg
@@ -133,15 +133,15 @@ indexinfo[].command[].indexname "exact2"
 indexinfo[].command[].command "lowercase"
 indexinfo[].command[].indexname "exact2"
 indexinfo[].command[].command "exact @@"
-indexinfo[].command[].indexname "experimental"
+indexinfo[].command[].indexname "bm25_field"
 indexinfo[].command[].command "index"
-indexinfo[].command[].indexname "experimental"
+indexinfo[].command[].indexname "bm25_field"
 indexinfo[].command[].command "lowercase"
-indexinfo[].command[].indexname "experimental"
+indexinfo[].command[].indexname "bm25_field"
 indexinfo[].command[].command "stem:BEST"
-indexinfo[].command[].indexname "experimental"
+indexinfo[].command[].indexname "bm25_field"
 indexinfo[].command[].command "normalize"
-indexinfo[].command[].indexname "experimental"
+indexinfo[].command[].indexname "bm25_field"
 indexinfo[].command[].command "plain-tokens"
 indexinfo[].command[].indexname "ia"
 indexinfo[].command[].command "index"
diff --git a/config-model/src/test/derived/indexschema/indexschema.cfg b/config-model/src/test/derived/indexschema/indexschema.cfg
index 998e53136a4..e8d064723da 100644
--- a/config-model/src/test/derived/indexschema/indexschema.cfg
+++ b/config-model/src/test/derived/indexschema/indexschema.cfg
@@ -78,7 +78,7 @@ indexfield[].phrases false
 indexfield[].positions true
 indexfield[].averageelementlen 512
 indexfield[].interleavedfeatures false
-indexfield[].name "experimental"
+indexfield[].name "bm25_field"
 indexfield[].datatype STRING
 indexfield[].collectiontype SINGLE
 indexfield[].prefix false
diff --git a/config-model/src/test/derived/indexschema/indexschema.sd b/config-model/src/test/derived/indexschema/indexschema.sd
index 44956f30e9e..49f0f7dfca6 100644
--- a/config-model/src/test/derived/indexschema/indexschema.sd
+++ b/config-model/src/test/derived/indexschema/indexschema.sd
@@ -56,9 +56,9 @@ search indexschema {
         exact
       }
     }
-    field experimental type string {
+    field bm25_field type string {
       indexing: index
-      index: experimental-posting-list-format
+      index: enable-bm25
     }
 
     # integer fields
diff --git a/config-model/src/test/derived/indexschema/vsmfields.cfg b/config-model/src/test/derived/indexschema/vsmfields.cfg
index 30ed67f61b7..9dcffd30313 100644
--- a/config-model/src/test/derived/indexschema/vsmfields.cfg
+++ b/config-model/src/test/derived/indexschema/vsmfields.cfg
@@ -55,7 +55,7 @@ fieldspec[].searchmethod AUTOUTF8
 fieldspec[].arg1 "exact"
 fieldspec[].maxlength 1048576
 fieldspec[].fieldtype INDEX
-fieldspec[].name "experimental"
+fieldspec[].name "bm25_field"
 fieldspec[].searchmethod AUTOUTF8
 fieldspec[].arg1 ""
 fieldspec[].maxlength 1048576
@@ -138,8 +138,8 @@ documenttype[].index[].name "exact1"
 documenttype[].index[].field[].name "exact1"
 documenttype[].index[].name "exact2"
 documenttype[].index[].field[].name "exact2"
-documenttype[].index[].name "experimental"
-documenttype[].index[].field[].name "experimental"
+documenttype[].index[].name "bm25_field"
+documenttype[].index[].field[].name "bm25_field"
 documenttype[].index[].name "ia"
 documenttype[].index[].field[].name "ia"
 documenttype[].index[].name "ib"