diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-03-23 17:03:28 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-03-23 17:03:28 +0100 |
| commit | f8cf0739f35de7b8da799e5206421ec5dc66df49 (patch) | |
| tree | 9820da3e8def61610631d1ca4e978a10471cc50f | |
| parent | 2da65b6a987716d59a27a720a03f34fc6c9a596b (diff) | |
| parent | f86aedaee7c84fff85bd97099b7e08db3e4d5313 (diff) | |
Merge pull request #17122 from vespa-engine/balder/orderting-2-type
Balder/orderting 2 type
46 files changed, 507 insertions, 84 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 3e421f9ba05..4a415fccbcc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -6,6 +6,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Dictionary; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; import com.yahoo.searchdefinition.document.Sorting; @@ -251,9 +252,25 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce ib.hnsw.multithreadedindexing(params.multiThreadedIndexing()); aaB.index(ib); } + Dictionary dictionary = attribute.getDictionary(); + if (dictionary != null) { + aaB.dictionary.type(convert(dictionary.getType())); + } return aaB; } + private static AttributesConfig.Attribute.Dictionary.Type.Enum convert(Dictionary.Type type) { + switch (type) { + case BTREE: + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + case HASH: + return AttributesConfig.Attribute.Dictionary.Type.HASH; + case BTREE_AND_HASH: + return AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH; + } + return AttributesConfig.Attribute.Dictionary.Type.BTREE; + } + public void getConfig(AttributesConfig.Builder builder, FieldSet fs) { for (Attribute attribute : attributes.values()) { if (isAttributeInFieldSet(attribute, fs)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java index 8cf862a72af..f230a7c10eb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -78,6 +78,8 @@ public final class Attribute implements Cloneable, Serializable { /** The aliases for this attribute */ private final Set<String> aliases = new LinkedHashSet<>(); + private Dictionary dictionary = new Dictionary(); + /** * True if this attribute should be returned during first pass of search. * Null means make the default decision for this kind of attribute @@ -208,6 +210,7 @@ public final class Attribute implements Cloneable, Serializable { public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; } public Sorting getSorting() { return sorting; } + public Dictionary getDictionary() { return dictionary; } public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; } @@ -231,6 +234,7 @@ public final class Attribute implements Cloneable, Serializable { public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } public void setDistanceMetric(DistanceMetric metric) { this.distanceMetric = Optional.of(metric); } public void setHnswIndexParams(HnswIndexParams params) { this.hnswIndexParams = Optional.of(params); } + public void setDictionary(Dictionary dictionary) { this.dictionary = dictionary; } public String getName() { return name; } public Type getType() { return type; } @@ -348,7 +352,7 @@ public final class Attribute implements Cloneable, Serializable { @Override public int hashCode() { return Objects.hash( - name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, + name, type, collectionType, sorting, dictionary, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent, isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); } @@ -370,10 +374,10 @@ public final class Attribute implements Cloneable, Serializable { if (this.createIfNonExistent != other.createIfNonExistent) return false; if (this.enableBitVectors != other.enableBitVectors) return false; if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; - // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now if (this.fastSearch != other.fastSearch) return false; if (this.huge != other.huge) return false; if (! this.sorting.equals(other.sorting)) return false; + if (! Objects.equals(dictionary, other.dictionary)) return false; if (! Objects.equals(tensorType, other.tensorType)) return false; if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false; if (! Objects.equals(distanceMetric, other.distanceMetric)) return false; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java new file mode 100644 index 00000000000..e492d572f27 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java @@ -0,0 +1,16 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.document; + +/** + * Represents settings for dictionary control + * + * @author baldersheim + */ +public class Dictionary { + public enum Type { BTREE, HASH, BTREE_AND_HASH }; + private final Type type; + public Dictionary() { this(Type.BTREE); } + public Dictionary(Type type) { this.type = type; } + public Type getType() { return type; } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index d81394382c2..76b707fa19b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -80,6 +80,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ private Matching matching = new Matching(); + private Dictionary dictionary = null; + /** Attribute settings, or null if there are none */ private final Map<String, Attribute> attributes = new TreeMap<>(); @@ -533,6 +535,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, public void setMatching(Matching matching) { this.matching=matching; } /** + * Returns Dictionary settings. + */ + public Dictionary getDictionary() { return dictionary; } + + + public void setDictionary(Dictionary dictionary) { this.dictionary=dictionary; } + + /** * Set the matching type for this field and all subfields. */ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java index b638932a4a8..56e241adb8e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -90,10 +90,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain this.enableOnlyBitVector = enableOnlyBitVector; } - public boolean isDoAlias() { - return doAlias; - } - public void setDoAlias(boolean doAlias) { this.doAlias = doAlias; } @@ -106,9 +102,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain this.alias = alias; } - public String getAliasedName() { - return aliasedName; - } public void setAliasedName(String aliasedName) { this.aliasedName = aliasedName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java new file mode 100644 index 00000000000..ce7c5a71a21 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.SDField; + +/** + * Represents operations controlling setup of dictionary used for queries + * + * @author baldersheim + */ +public class DictionaryOperation implements FieldOperation { + private final Dictionary.Type type; + + public DictionaryOperation(Dictionary.Type type) { + this.type = type; + } + @Override + public void apply(SDField field) { + Dictionary prev = field.getDictionary(); + if (prev == null) { + field.setDictionary(new Dictionary(type)); + } else if ((prev.getType() == Dictionary.Type.BTREE && type == Dictionary.Type.HASH) || + (prev.getType() == Dictionary.Type.HASH && type == Dictionary.Type.BTREE)) + { + field.setDictionary(new Dictionary(Dictionary.Type.BTREE_AND_HASH)); + } else { + if (prev.getType() != type) { + throw new IllegalArgumentException("Can not combine previous dictionary setting " + prev.getType() + + " with current " + type); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java new file mode 100644 index 00000000000..92ceb8e5e44 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.NumericDataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Propagates dictionary settings from field level to attribute level. + * Only applies to numeric fields with fast-search enabled. + * + * @author baldersheim + */ +public class DictionaryProcessor extends Processor { + public DictionaryProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + @Override + public void process(boolean validate, boolean documentsOnly) { + for (SDField field : search.allConcreteFields()) { + Dictionary dictionary = field.getDictionary(); + if (dictionary == null) continue; + + Attribute attribute = field.getAttribute(); + if (attribute.getDataType() instanceof NumericDataType ) { + if (attribute.isFastSearch()) { + attribute.setDictionary(dictionary); + } else { + fail(search, field, "You must specify 'attribute:fast-search' to allow dictionary control"); + } + } else { + fail(search, field, "You can only specify 'dictionary:' for numeric fields"); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 79f19efe422..ff0edcd0404 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -15,7 +15,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; -import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.ArrayList; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 1a3ef9e54b4..136d352ece7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -37,6 +37,7 @@ public class Processing { AttributesImplicitWord::new, MutableAttributes::new, CreatePositionZCurve::new, + DictionaryProcessor::new, WordMatch::new, ImportedFieldsResolver::new, ImplicitSummaries::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index 3744af7cc2c..61b5e6f2a64 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -25,7 +25,7 @@ import java.util.logging.Level; public abstract class Processor { protected final Search search; - protected DeployLogger deployLogger; + protected final DeployLogger deployLogger; protected final RankProfileRegistry rankProfileRegistry; protected final QueryProfiles queryProfiles; diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 869f671d8ef..2607b2fb02b 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -252,6 +252,7 @@ TOKEN : | < PROPERTIES: "properties" > | < ATTRIBUTE: "attribute" > | < SORTING: "sorting" > +| < DICTIONARY: "dictionary" > | < ASCENDING: "ascending" > | < DESCENDING: "descending" > | < UCA: "uca" > @@ -267,6 +268,8 @@ TOKEN : | < IDENTICAL: "identical" > | < STEMMING: "stemming" > | < NORMALIZING: "normalizing" > +| < HASH: "hash" > +| < BTREE: "btree" > | < BOLDING: "bolding" > | < BODY: "body" > | < HEADER: "header" > @@ -987,6 +990,7 @@ String fieldBody(SDField field, Search search, SDDocumentType document) : { } attribute(field) | body(field) | bolding(field) | + dictionary(field) | fieldStemming(field) | header(field) | id(field, document) | @@ -1520,6 +1524,34 @@ void bolding(FieldOperationContainer field) : } /** + * This rule consumes a dictionary statement of a field element. + * + * @param field The field to modify. + */ +void dictionary(FieldOperationContainer field) : +{ + Dictionary.Type type; +} +{ + <DICTIONARY> <COLON> type = dictionaryType() + { + field.addOperation(new DictionaryOperation(type)); + } +} + +Dictionary.Type dictionaryType() : +{ + Dictionary.Type type; +} +{ + ( <HASH> { type = Dictionary.Type.HASH; } + | <BTREE> { type = Dictionary.Type.BTREE; } ) + { + return type; + } +} + +/** * This rule consumes a body statement of a field element. * * @param field The field to modify. diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 0d4b3bc4ca7..641e2d8fde5 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "location_zcurve" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index dc269284e8f..bb4ba665406 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "elem_array.name" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -30,6 +31,7 @@ attribute[].name "elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index c43e38d0561..d05d2d1d5e0 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "singlebyte" attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "multibyte" attribute[].datatype INT8 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "wsbyte" attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "singleint" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "multiint" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "wsint" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "singlelong" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "multilong" attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "wslong" attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "singlefloat" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "multifloat" attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -310,6 +321,7 @@ attribute[].name "wsfloat" attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -338,6 +350,7 @@ attribute[].name "singledouble" attribute[].datatype DOUBLE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -366,6 +379,7 @@ attribute[].name "multidouble" attribute[].datatype DOUBLE attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -394,6 +408,7 @@ attribute[].name "wsdouble" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -422,6 +437,7 @@ attribute[].name "singlestring" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -450,6 +466,7 @@ attribute[].name "multistring" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -478,6 +495,7 @@ attribute[].name "wsstring" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index 4ecb0c2af8f..e9f3f68adc1 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "a1" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "a2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "a3" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "a5" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "a6" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "b1" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "b2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "b3" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "b4" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "b5" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "b6" attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -310,6 +321,7 @@ attribute[].name "b7" attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -338,6 +350,7 @@ attribute[].name "a9" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -366,6 +379,7 @@ attribute[].name "a10" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -394,6 +408,7 @@ attribute[].name "a11" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -422,6 +437,7 @@ attribute[].name "a12" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -450,6 +466,7 @@ attribute[].name "a7_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -478,6 +495,7 @@ attribute[].name "a8_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index fe6f42e55bc..622fb9f349c 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "prefixenabled" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "fleeting" attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "fleeting2" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "foundat" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "collapseby" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "ts" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "combineda" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "year_arr" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "year_sub" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index 53eb98b6ba8..4d275787bfd 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "t1" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "t2" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg index 2c209db96f2..bfdf90ac12c 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "ref_from_a" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "ref_from_b" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "from_a_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "from_b_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index 5bb0d9bfef3..f1b20c6e454 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "parent_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "my_pos_zcurve" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index f299756efb1..9e0b5f18170 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "parent_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "my_elem_array.name" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -58,6 +60,7 @@ attribute[].name "my_elem_array.weight" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "my_elem_map.key" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -114,6 +118,7 @@ attribute[].name "my_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -142,6 +147,7 @@ attribute[].name "my_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "my_str_int_map.key" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -198,6 +205,7 @@ attribute[].name "my_str_int_map.value" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 680d6571939..68cd917336f 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "a_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "b_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "my_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "my_string_field" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "my_int_array_field" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "my_int_wset_field" attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "my_ancient_int_field" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 52367a04a30..a931af7af4d 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "onlygrandparent" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "overridden" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "onlymother" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index b25a46dc433..11498de54b1 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index d9624e89500..acbdf119d0d 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "str_map.key" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -30,6 +31,7 @@ attribute[].name "str_map.value" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "int_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 5fd10269ceb..ecc8c2fd69d 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "str_elem_map.key" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -30,6 +31,7 @@ attribute[].name "str_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "int_elem_map.key" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "int_elem_map.value.name" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index ebb8eb10c14..a31325e67d0 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "sales" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "hiphopvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "metalvalue_arr" attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index 5f5e7e62d83..68f24871f02 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "sales" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "pto" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "mid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "weight" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "bgnpfrom" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "newestedition" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "year" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "did" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "scorekey" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "cbid" attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 6a7ff8af7ad..ff45cf1a41e 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "some_predicate_field" attribute[].datatype PREDICATE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index 2a5fc890f9e..f9878068372 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "attributefield1" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "attributefield2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index 3ecd24d50dc..ad3ff5e62f8 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "campaign_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "other_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "yet_another_ref" attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index a84c7780965..ebe0e83540e 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "syntaxcheck" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "syntaxcheck2" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "infieldonly" attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index cc28f1b6f84..398cdf5f8f8 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "f2" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "f3" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "f4" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "f5" attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "f6" attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index d30c79efbc2..05e19a15d96 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -2,6 +2,7 @@ attribute[].name "abyte" attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -30,6 +31,7 @@ attribute[].name "along" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -58,6 +60,7 @@ attribute[].name "abool" attribute[].datatype BOOL attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -86,6 +89,7 @@ attribute[].name "ashortfloat" attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -114,6 +118,7 @@ attribute[].name "arrayfield" attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -142,6 +147,7 @@ attribute[].name "setfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -170,6 +176,7 @@ attribute[].name "setfield2" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -198,6 +205,7 @@ attribute[].name "setfield3" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent false attribute[].fastsearch false @@ -226,6 +234,7 @@ attribute[].name "setfield4" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent true attribute[].fastsearch false @@ -254,6 +263,7 @@ attribute[].name "tagfield" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -282,6 +292,7 @@ attribute[].name "juletre" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -310,6 +321,7 @@ attribute[].name "album1" attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -338,6 +350,7 @@ attribute[].name "other" attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED +attribute[].dictionary.type BTREE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 911f8e797e1..5589ad018a7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.SearchBuilder; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java new file mode 100644 index 00000000000..256858b372e --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java @@ -0,0 +1,125 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.model.test.TestUtil; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Test configuration of dictionary control. + * + * @author baldersheim + */ +public class DictionaryTestCase { + private static AttributesConfig getConfig(Search search) { + AttributeFields attributes = new AttributeFields(search); + AttributesConfig.Builder builder = new AttributesConfig.Builder(); + attributes.getConfig(builder); + return builder.build(); + } + private Search createSearch(String def) throws ParseException { + SearchBuilder sb = SearchBuilder.createFromString(def); + return sb.getSearch(); + } + @Test + public void testDefaultDictionarySettings() throws ParseException { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field s1 type string {", + " indexing: attribute | summary", + " }", + " field n1 type int {", + " indexing: summary | attribute", + " }", + " }", + "}"); + Search search = createSearch(def); + assertEquals(Dictionary.Type.BTREE, search.getAttribute("s1").getDictionary().getType()); + assertEquals(Dictionary.Type.BTREE, search.getAttribute("n1").getDictionary().getType()); + } + + void verifyNumericDictionaryControl(Dictionary.Type expected, + AttributesConfig.Attribute.Dictionary.Type.Enum expectedConfig, + String ... cfg) throws ParseException + { + String def = TestUtil.joinLines( + "search test {", + " document test {", + " field n1 type int {", + " indexing: summary | attribute", + " attribute:fast-search", + TestUtil.joinLines(cfg), + " }", + " }", + "}"); + Search search = createSearch(def); + assertEquals(expected, search.getAttribute("n1").getDictionary().getType()); + assertEquals(expectedConfig, + getConfig(search).attribute().get(0).dictionary().type()); + } + + @Test + public void testNumericBtreeSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.BTREE, + AttributesConfig.Attribute.Dictionary.Type.BTREE, + "dictionary:btree"); + } + @Test + public void testNumericHashSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.HASH, + AttributesConfig.Attribute.Dictionary.Type.HASH, + "dictionary:hash"); + } + @Test + public void testNumericBtreeAndHashSettings() throws ParseException { + verifyNumericDictionaryControl(Dictionary.Type.BTREE_AND_HASH, + AttributesConfig.Attribute.Dictionary.Type.BTREE_AND_HASH, + "dictionary:btree", "dictionary:hash"); + } + @Test + public void testNonNumericFieldsFailsDictionaryControl() throws ParseException { + String def = + "search test {\n" + + " document test {\n" + + " field n1 type string {\n" + + " indexing: summary | attribute\n" + + " dictionary:btree\n" + + " }\n" + + " }\n" + + "}\n"; + try { + SearchBuilder sb = SearchBuilder.createFromString(def); + fail("Controlling dictionary for non-numeric fields are not yet supported."); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': You can only specify 'dictionary:' for numeric fields", e.getMessage()); + } + } + @Test + public void testNonFastSearchFieldsFailsDictionaryControl() throws ParseException { + String def = + "search test {\n" + + " document test {\n" + + " field n1 type int {\n" + + " indexing: summary | attribute\n" + + " dictionary:btree\n" + + " }\n" + + " }\n" + + "}\n"; + try { + SearchBuilder sb = SearchBuilder.createFromString(def); + fail("Controlling dictionary for non-fast-search fields are not allowed."); + } catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'n1': You must specify 'attribute:fast-search' to allow dictionary control", e.getMessage()); + } + } +} diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index be336c22cb6..939a0b8476a 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -4,7 +4,9 @@ namespace=vespa.config.search attribute[].name string attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +# Deprecated/ do-not-use, will soon be GCed. attribute[].dictionary.ordering enum { ORDERED, UNORDERED } default = ORDERED +attribute[].dictionary.type enum { BTREE, HASH, BTREE_AND_HASH } default = BTREE attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false attribute[].fastsearch bool default=false diff --git a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp index 3dc1cf6d27e..98b2bfe5c90 100644 --- a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp +++ b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp @@ -110,22 +110,24 @@ TEST("Test GrowStrategy consistency") { } TEST("DictionaryConfig") { - using Ordering = DictionaryConfig::Ordering; - EXPECT_EQUAL(Ordering::ORDERED, DictionaryConfig().getOrdering()); - EXPECT_EQUAL(Ordering::ORDERED, DictionaryConfig(Ordering::ORDERED).getOrdering()); - EXPECT_EQUAL(Ordering::UNORDERED, DictionaryConfig(Ordering::UNORDERED).getOrdering()); - EXPECT_EQUAL(DictionaryConfig(Ordering::ORDERED), DictionaryConfig(Ordering::ORDERED)); - EXPECT_EQUAL(DictionaryConfig(Ordering::UNORDERED), DictionaryConfig(Ordering::UNORDERED)); - EXPECT_NOT_EQUAL(DictionaryConfig(Ordering::UNORDERED), DictionaryConfig(Ordering::ORDERED)); - EXPECT_NOT_EQUAL(DictionaryConfig(Ordering::ORDERED), DictionaryConfig(Ordering::UNORDERED)); - EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) == - Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED))); - EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) == - Config().set_dictionary_config(DictionaryConfig(Ordering::ORDERED))); - EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) != - Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED))); - EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) != - Config().set_dictionary_config(DictionaryConfig(Ordering::ORDERED))); + using Type = DictionaryConfig::Type; + EXPECT_EQUAL(Type::BTREE, DictionaryConfig().getType()); + EXPECT_EQUAL(Type::BTREE, DictionaryConfig(Type::BTREE).getType()); + EXPECT_EQUAL(Type::HASH, DictionaryConfig(Type::HASH).getType()); + EXPECT_EQUAL(Type::BTREE_AND_HASH, DictionaryConfig(Type::BTREE_AND_HASH).getType()); + EXPECT_EQUAL(DictionaryConfig(Type::BTREE), DictionaryConfig(Type::BTREE)); + EXPECT_EQUAL(DictionaryConfig(Type::HASH), DictionaryConfig(Type::HASH)); + EXPECT_EQUAL(DictionaryConfig(Type::BTREE_AND_HASH), DictionaryConfig(Type::BTREE_AND_HASH)); + EXPECT_NOT_EQUAL(DictionaryConfig(Type::HASH), DictionaryConfig(Type::BTREE)); + EXPECT_NOT_EQUAL(DictionaryConfig(Type::BTREE), DictionaryConfig(Type::HASH)); + EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) == + Config().set_dictionary_config(DictionaryConfig(Type::HASH))); + EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) == + Config().set_dictionary_config(DictionaryConfig(Type::BTREE))); + EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) != + Config().set_dictionary_config(DictionaryConfig(Type::HASH))); + EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Type::HASH)) != + Config().set_dictionary_config(DictionaryConfig(Type::BTREE))); } diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp b/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp index 00b6ae2710f..a6a0255f96d 100644 --- a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp +++ b/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp @@ -8,11 +8,13 @@ namespace search { std::ostream& operator<<(std::ostream& os, const DictionaryConfig & cfg) { - switch(cfg.getOrdering()) { - case DictionaryConfig::Ordering::ORDERED: - return os << "ORDERED"; - case DictionaryConfig::Ordering::UNORDERED: - return os << "UNORDERED"; + switch(cfg.getType()) { + case DictionaryConfig::Type::BTREE: + return os << "BTREE"; + case DictionaryConfig::Type::HASH: + return os << "HASH"; + case DictionaryConfig::Type::BTREE_AND_HASH: + return os << "BTREE_AND_HASH"; } assert(false); } diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h b/searchcommon/src/vespa/searchcommon/common/dictionary_config.h index 06c88d88670..c35f7eaafef 100644 --- a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h +++ b/searchcommon/src/vespa/searchcommon/common/dictionary_config.h @@ -11,13 +11,13 @@ namespace search { */ class DictionaryConfig { public: - enum class Ordering { ORDERED, UNORDERED }; - DictionaryConfig() noexcept : _ordering(Ordering::ORDERED) {} - DictionaryConfig(Ordering ordering) noexcept : _ordering(ordering) {} - Ordering getOrdering() const { return _ordering; } - bool operator == (const DictionaryConfig & b) const { return _ordering == b._ordering; } + enum class Type { BTREE, HASH, BTREE_AND_HASH }; + DictionaryConfig() noexcept : _type(Type::BTREE) {} + DictionaryConfig(Type ordering) noexcept : _type(ordering) {} + Type getType() const { return _type; } + bool operator == (const DictionaryConfig & b) const { return _type == b._type; } private: - Ordering _ordering; + Type _type; }; std::ostream& operator<<(std::ostream& os, const DictionaryConfig & cfg); diff --git a/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp b/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp index 3690533eef9..d999a6f37a2 100644 --- a/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp +++ b/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp @@ -27,7 +27,7 @@ using NodeAllocator = TreeType::NodeAllocatorType; TEST("requireThatNumericLessIsWorking") { - NumericEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10); EnumIndex e2 = es.insert(30); auto cmp1 = es.make_comparator(); @@ -41,7 +41,7 @@ TEST("requireThatNumericLessIsWorking") TEST("requireThatNumericEqualIsWorking") { - NumericEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10); EnumIndex e2 = es.insert(30); auto cmp1 = es.make_comparator(); @@ -56,7 +56,7 @@ TEST("requireThatNumericEqualIsWorking") TEST("requireThatFloatLessIsWorking") { - FloatEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + FloatEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10.5); EnumIndex e2 = es.insert(30.5); EnumIndex e3 = es.insert(std::numeric_limits<float>::quiet_NaN()); @@ -74,7 +74,7 @@ TEST("requireThatFloatLessIsWorking") TEST("requireThatFloatEqualIsWorking") { - FloatEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + FloatEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert(10.5); EnumIndex e2 = es.insert(30.5); EnumIndex e3 = es.insert(std::numeric_limits<float>::quiet_NaN()); @@ -93,7 +93,7 @@ TEST("requireThatFloatEqualIsWorking") TEST("requireThatStringLessIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); @@ -110,7 +110,7 @@ TEST("requireThatStringLessIsWorking") TEST("requireThatStringEqualIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); @@ -127,7 +127,7 @@ TEST("requireThatStringEqualIsWorking") TEST("requireThatComparatorWithTreeIsWorking") { - NumericEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore es(false, DictionaryConfig::Type::BTREE); vespalib::GenerationHandler g; TreeType t; NodeAllocator m; @@ -152,7 +152,7 @@ TEST("requireThatComparatorWithTreeIsWorking") TEST("requireThatFoldedLessIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); @@ -172,7 +172,7 @@ TEST("requireThatFoldedLessIsWorking") TEST("requireThatFoldedEqualIsWorking") { - StringEnumStore es(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 769a0d86343..8ce3de52521 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -6,7 +6,7 @@ #include <vespa/log/log.h> LOG_SETUP("enumstore_test"); -using Ordering = search::DictionaryConfig::Ordering; +using Type = search::DictionaryConfig::Type; using vespalib::datastore::EntryRef; namespace search { @@ -19,42 +19,42 @@ using StringEnumStore = EnumStoreT<const char*>; struct OrderedDoubleEnumStore { using EnumStoreType = DoubleEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedDoubleEnumStore { using EnumStoreType = DoubleEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; struct OrderedFloatEnumStore { using EnumStoreType = FloatEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedFloatEnumStore { using EnumStoreType = FloatEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; struct OrderedNumericEnumStore { using EnumStoreType = NumericEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedNumericEnumStore { using EnumStoreType = NumericEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; struct OrderedStringEnumStore { using EnumStoreType = StringEnumStore; - static constexpr Ordering ordering = Ordering::ORDERED; + static constexpr Type type = Type::BTREE; }; struct UnorderedStringEnumStore { using EnumStoreType = StringEnumStore; - static constexpr Ordering ordering = Ordering::UNORDERED; + static constexpr Type type = Type::BTREE_AND_HASH; }; using StringVector = std::vector<std::string>; @@ -106,7 +106,7 @@ public: using EnumStoreType = typename EnumStoreTypeAndOrdering::EnumStoreType; EnumStoreType es; FloatEnumStoreTest() - : es(false, EnumStoreTypeAndOrdering::ordering) + : es(false, EnumStoreTypeAndOrdering::type) {} }; @@ -150,7 +150,7 @@ TYPED_TEST(FloatEnumStoreTest, numbers_can_be_inserted_and_retrieved) TEST(EnumStoreTest, test_find_folded_on_string_enum_store) { - StringEnumStore ses(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore ses(false, DictionaryConfig::Type::BTREE); std::vector<EnumIndex> indices; std::vector<std::string> unique({"", "one", "two", "TWO", "Two", "three"}); for (std::string &str : unique) { @@ -201,7 +201,7 @@ public: void StringEnumStoreTest::testInsert(bool hasPostings) { - StringEnumStore ses(hasPostings, DictionaryConfig::Ordering::ORDERED); + StringEnumStore ses(hasPostings, DictionaryConfig::Type::BTREE); std::vector<EnumIndex> indices; std::vector<std::string> unique; @@ -251,7 +251,7 @@ TEST_F(StringEnumStoreTest, test_insert_on_store_with_posting_lists) TEST(EnumStoreTest, test_hold_lists_and_generation) { - StringEnumStore ses(false, DictionaryConfig::Ordering::ORDERED); + StringEnumStore ses(false, DictionaryConfig::Type::BTREE); StringVector uniques; generation_t sesGen = 0u; uniques.reserve(100); @@ -328,7 +328,7 @@ dec_ref_count(NumericEnumStore& store, NumericEnumStore::Index idx) TEST(EnumStoreTest, address_space_usage_is_reported) { const size_t ADDRESS_LIMIT = 4290772994; // Max allocated elements in un-allocated buffers + allocated elements in allocated buffers. - NumericEnumStore store(false, DictionaryConfig::Ordering::ORDERED); + NumericEnumStore store(false, DictionaryConfig::Type::BTREE); using vespalib::AddressSpace; EXPECT_EQ(AddressSpace(1, 1, ADDRESS_LIMIT), store.get_address_space_usage()); @@ -350,7 +350,7 @@ public: EnumIndex i5; BatchUpdaterTest() - : store(false, DictionaryConfig::Ordering::ORDERED), + : store(false, DictionaryConfig::Type::BTREE), i3(), i5() { @@ -458,7 +458,7 @@ public: using Values = LoaderTestValues<EnumStoreType>; LoaderTest() - : store(true, EnumStoreTypeAndOrdering::ordering) + : store(true, EnumStoreTypeAndOrdering::type) {} void load_values(enumstore::EnumeratedLoaderBase& loader) const { @@ -570,7 +570,7 @@ public: EnumStoreType store; EnumStoreDictionaryTest() - : store(true, EnumStoreTypeAndOrdering::ordering) + : store(true, EnumStoreTypeAndOrdering::type) {} // Reuse test values from LoaderTest @@ -618,7 +618,7 @@ TYPED_TEST(EnumStoreDictionaryTest, find_frozen_index_works) this->update_posting_idx(value_0_idx, EntryRef(), this->fake_pidx()); auto& dict = this->store.get_dictionary(); EnumIndex idx; - if (TypeParam::ordering == Ordering::ORDERED) { + if (TypeParam::type == Type::BTREE) { EXPECT_FALSE(dict.find_frozen_index(this->make_bound_comparator(0), idx)); } else { EXPECT_TRUE(dict.find_frozen_index(this->make_bound_comparator(0), idx)); @@ -640,7 +640,7 @@ TYPED_TEST(EnumStoreDictionaryTest, find_posting_list_works) auto& dict = this->store.get_dictionary(); auto root = dict.get_frozen_root(); auto find_result = dict.find_posting_list(this->make_bound_comparator(0), root); - if (TypeParam::ordering == Ordering::ORDERED) { + if (TypeParam::type == Type::BTREE) { EXPECT_FALSE(find_result.first.valid()); EXPECT_FALSE(find_result.second.valid()); } else { diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 145a021801d..2e946c7d34b 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -50,20 +50,22 @@ getCollectionTypeMap() static DataTypeMap _dataTypeMap = getDataTypeMap(); static CollectionTypeMap _collectionTypeMap = getCollectionTypeMap(); -DictionaryConfig::Ordering -convert(AttributesConfig::Attribute::Dictionary::Ordering ordering_cfg) { - switch (ordering_cfg) { - case AttributesConfig::Attribute::Dictionary::Ordering::ORDERED: - return DictionaryConfig::Ordering::ORDERED; - case AttributesConfig::Attribute::Dictionary::Ordering::UNORDERED: - return DictionaryConfig::Ordering::UNORDERED; +DictionaryConfig::Type +convert(AttributesConfig::Attribute::Dictionary::Type type_cfg) { + switch (type_cfg) { + case AttributesConfig::Attribute::Dictionary::Type::BTREE: + return DictionaryConfig::Type::BTREE; + case AttributesConfig::Attribute::Dictionary::Type::HASH: + return DictionaryConfig::Type::HASH; + case AttributesConfig::Attribute::Dictionary::Type::BTREE_AND_HASH: + return DictionaryConfig::Type::BTREE_AND_HASH; } assert(false); } DictionaryConfig convert_dictionary(const AttributesConfig::Attribute::Dictionary & dictionary) { - return DictionaryConfig(convert(dictionary.ordering)); + return DictionaryConfig(convert(dictionary.type)); } } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index dc501dc9d89..fd523e227b0 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -13,7 +13,7 @@ EnumAttribute<B>:: EnumAttribute(const vespalib::string &baseFileName, const AttributeVector::Config &cfg) : B(baseFileName, cfg), - _enumStore(cfg.fastSearch(), cfg.get_dictionary_config().getOrdering()) + _enumStore(cfg.fastSearch(), cfg.get_dictionary_config().getType()) { this->setEnum(true); } diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 0961f7c87f1..5beafea0046 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -42,14 +42,15 @@ EnumStoreT<const char*>::load_unique_value(const void* src, } std::unique_ptr<vespalib::datastore::IUniqueStoreDictionary> -make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Ordering ordering, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare) +make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Type type, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare) { if (has_postings) { if (folded_compare) { return std::make_unique<EnumStoreFoldedDictionary>(store, std::move(compare), std::move(folded_compare)); } else { - switch (ordering) { - case search::DictionaryConfig::Ordering::UNORDERED: + switch (type) { + case search::DictionaryConfig::Type::HASH: + case search::DictionaryConfig::Type::BTREE_AND_HASH: return std::make_unique<EnumStoreDictionary<EnumPostingTree, vespalib::datastore::SimpleHashMap>>(store, std::move(compare)); default: return std::make_unique<EnumStoreDictionary<EnumPostingTree>>(store, std::move(compare)); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index dc75ac1f38b..02b2ec019f6 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -74,8 +74,8 @@ private: ssize_t load_unique_value(const void* src, size_t available, Index& idx); public: - EnumStoreT(bool has_postings, search::DictionaryConfig::Ordering ordering); - virtual ~EnumStoreT(); + EnumStoreT(bool has_postings, search::DictionaryConfig::Type type); + ~EnumStoreT() override; uint32_t get_ref_count(Index idx) const { return get_entry_base(idx).get_ref_count(); } void inc_ref_count(Index idx) { return get_entry_base(idx).inc_ref_count(); } @@ -213,7 +213,9 @@ public: }; std::unique_ptr<vespalib::datastore::IUniqueStoreDictionary> -make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Ordering ordering, std::unique_ptr<vespalib::datastore::EntryComparator> compare, std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare); +make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::DictionaryConfig::Type type, + std::unique_ptr<vespalib::datastore::EntryComparator> compare, + std::unique_ptr<vespalib::datastore::EntryComparator> folded_compare); template <> diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 22f0d3e50dc..357026ab944 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -72,13 +72,13 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& } template <typename EntryT> -EnumStoreT<EntryT>::EnumStoreT(bool has_postings, search::DictionaryConfig::Ordering ordering) +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, search::DictionaryConfig::Type type) : _store(), _dict(), _cached_values_memory_usage(), _cached_values_address_space_usage(0, 0, (1ull << 32)) { - _store.set_dictionary(make_enum_store_dictionary(*this, has_postings, ordering, + _store.set_dictionary(make_enum_store_dictionary(*this, has_postings, type, std::make_unique<ComparatorType>(_store.get_data_store()), (has_string_type() ? std::make_unique<FoldedComparatorType>(_store.get_data_store()) : |