diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-06-25 16:26:49 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-25 16:26:49 +0200 |
commit | c3223ae0b3fcc9d36a0604da1f54abce19d6a135 (patch) | |
tree | 492d9ec28a4fa636e636a727f646cb75f64bdd22 | |
parent | 1c5fd4508cac4cb1d649a729531fbe49f1d37dfa (diff) | |
parent | 706e6cb32b33fe718ef1fc7325bc78e06b8c84fb (diff) |
Merge pull request #9898 from vespa-engine/geirst/flag-for-bm25-posting-lists
Rename flag to turn on new posting list format with interleaved featu…
8 files changed, 29 insertions, 30 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index d7e9e0da081..0ea3f5c24a3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -56,8 +56,8 @@ public class Index implements Cloneable, Serializable { /** The boolean index definition, if set */ private BooleanIndexDefinition boolIndex; - // TODO: Remove when experimental posting list format is made default - private boolean experimentalPostingListFormat = false; + /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */ + private boolean interleavedFeatures = false; public Index(String name) { this(name, false); @@ -184,12 +184,12 @@ public class Index implements Cloneable, Serializable { boolIndex = def; } - public void setExperimentalPostingListFormat(boolean value) { - experimentalPostingListFormat = value; + public void setInterleavedFeatures(boolean value) { + interleavedFeatures = value; } - public boolean useExperimentalPostingListFormat() { - return experimentalPostingListFormat; + public boolean useInterleavedFeatures() { + return interleavedFeatures; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index 3b62807ce73..60b8ee78c7b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -114,7 +114,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { .prefix(f.hasPrefix()) .phrases(f.hasPhrases()) .positions(f.hasPositions()) - .interleavedfeatures(f.useExperimentalPostingListFormat()); + .interleavedfeatures(f.useInterleavedFeatures()); if (!f.getCollectionType().equals("SINGLE")) { ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); } @@ -175,8 +175,8 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility? private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility? private BooleanIndexDefinition boolIndex = null; - // TODO: Remove when experimental posting list format is made default - private boolean experimentalPostingListFormat = false; + // Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. + private boolean interleavedFeatures = false; public IndexField(String name, Index.Type type, DataType sdFieldType) { this.name = name; @@ -186,7 +186,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { public void setIndexSettings(com.yahoo.searchdefinition.Index index) { if (type.equals(Index.Type.TEXT)) { prefix = index.isPrefix(); - experimentalPostingListFormat = index.useExperimentalPostingListFormat(); + interleavedFeatures = index.useInterleavedFeatures(); } sdType = index.getType(); boolIndex = index.getBooleanIndexDefiniton(); @@ -209,7 +209,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { public boolean hasPrefix() { return prefix; } public boolean hasPhrases() { return phrases; } public boolean hasPositions() { return positions; } - public boolean useExperimentalPostingListFormat() { return experimentalPostingListFormat; } + public boolean useInterleavedFeatures() { return interleavedFeatures; } public BooleanIndexDefinition getBooleanIndexDefinition() { return boolIndex; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java index 459bb247e5f..39f543c7db3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -29,8 +29,7 @@ public class IndexOperation implements FieldOperation { private OptionalLong lowerBound = OptionalLong.empty(); private OptionalLong upperBound = OptionalLong.empty(); private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); - // TODO: Remove when experimental posting list format is made default - private Optional<Boolean> experimentalPostingListFormat = Optional.empty(); + private Optional<Boolean> enableBm25 = Optional.empty(); public String getIndexName() { return indexName; @@ -89,8 +88,8 @@ public class IndexOperation implements FieldOperation { index.setBooleanIndexDefiniton( new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold)); } - if (experimentalPostingListFormat.isPresent()) { - index.setExperimentalPostingListFormat(experimentalPostingListFormat.get()); + if (enableBm25.isPresent()) { + index.setInterleavedFeatures(enableBm25.get()); } } @@ -117,8 +116,8 @@ public class IndexOperation implements FieldOperation { public void setDensePostingListThreshold(double densePostingListThreshold) { this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); } - public void setExperimentalPostingListFormat(boolean value) { - experimentalPostingListFormat = Optional.of(value); + public void setEnableBm25(boolean value) { + enableBm25 = Optional.of(value); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 571ad452b01..6dde12f0fac 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -331,7 +331,7 @@ TOKEN : | < LOWERBOUND: "lower-bound" > | < UPPERBOUND: "upper-bound" > | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > -| < EXPERIMENTALPOSTINGLISTFORMAT: "experimental-posting-list-format" > +| < ENABLE_BM25: "enable-bm25" > | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? > @@ -1782,7 +1782,7 @@ Object indexBody(IndexOperation index) : | <LOWERBOUND> <COLON> num = consumeLong() { index.setLowerBound(num); } | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } - | <EXPERIMENTALPOSTINGLISTFORMAT> { index.setExperimentalPostingListFormat(true); } + | <ENABLE_BM25> { index.setEnableBm25(true); } ) { return null; } } diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index 46c2c3fc307..a83ec45c5e9 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -133,15 +133,15 @@ indexinfo[].command[].indexname "exact2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "exact2" indexinfo[].command[].command "exact @@" -indexinfo[].command[].indexname "experimental" +indexinfo[].command[].indexname "bm25_field" indexinfo[].command[].command "index" -indexinfo[].command[].indexname "experimental" +indexinfo[].command[].indexname "bm25_field" indexinfo[].command[].command "lowercase" -indexinfo[].command[].indexname "experimental" +indexinfo[].command[].indexname "bm25_field" indexinfo[].command[].command "stem:BEST" -indexinfo[].command[].indexname "experimental" +indexinfo[].command[].indexname "bm25_field" indexinfo[].command[].command "normalize" -indexinfo[].command[].indexname "experimental" +indexinfo[].command[].indexname "bm25_field" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "ia" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/indexschema/indexschema.cfg b/config-model/src/test/derived/indexschema/indexschema.cfg index 998e53136a4..e8d064723da 100644 --- a/config-model/src/test/derived/indexschema/indexschema.cfg +++ b/config-model/src/test/derived/indexschema/indexschema.cfg @@ -78,7 +78,7 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].interleavedfeatures false -indexfield[].name "experimental" +indexfield[].name "bm25_field" indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false diff --git a/config-model/src/test/derived/indexschema/indexschema.sd b/config-model/src/test/derived/indexschema/indexschema.sd index 44956f30e9e..49f0f7dfca6 100644 --- a/config-model/src/test/derived/indexschema/indexschema.sd +++ b/config-model/src/test/derived/indexschema/indexschema.sd @@ -56,9 +56,9 @@ search indexschema { exact } } - field experimental type string { + field bm25_field type string { indexing: index - index: experimental-posting-list-format + index: enable-bm25 } # integer fields diff --git a/config-model/src/test/derived/indexschema/vsmfields.cfg b/config-model/src/test/derived/indexschema/vsmfields.cfg index 30ed67f61b7..9dcffd30313 100644 --- a/config-model/src/test/derived/indexschema/vsmfields.cfg +++ b/config-model/src/test/derived/indexschema/vsmfields.cfg @@ -55,7 +55,7 @@ fieldspec[].searchmethod AUTOUTF8 fieldspec[].arg1 "exact" fieldspec[].maxlength 1048576 fieldspec[].fieldtype INDEX -fieldspec[].name "experimental" +fieldspec[].name "bm25_field" fieldspec[].searchmethod AUTOUTF8 fieldspec[].arg1 "" fieldspec[].maxlength 1048576 @@ -138,8 +138,8 @@ documenttype[].index[].name "exact1" documenttype[].index[].field[].name "exact1" documenttype[].index[].name "exact2" documenttype[].index[].field[].name "exact2" -documenttype[].index[].name "experimental" -documenttype[].index[].field[].name "experimental" +documenttype[].index[].name "bm25_field" +documenttype[].index[].field[].name "bm25_field" documenttype[].index[].name "ia" documenttype[].index[].field[].name "ia" documenttype[].index[].name "ib" |