diff options
10 files changed, 112 insertions, 21 deletions
diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 6c69d71fdf6..be336c22cb6 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -4,6 +4,7 @@ namespace=vespa.config.search attribute[].name string attribute[].datatype enum { STRING, BOOL, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE +attribute[].dictionary.ordering enum { ORDERED, UNORDERED } default = ORDERED attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false attribute[].fastsearch bool default=false diff --git a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp index 99de79b8976..3dc1cf6d27e 100644 --- a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp +++ b/searchcommon/src/tests/attribute/config/attribute_config_test.cpp @@ -8,6 +8,7 @@ using search::attribute::BasicType; using search::attribute::CollectionType; using vespalib::eval::ValueType; using search::GrowStrategy; +using search::DictionaryConfig; struct Fixture @@ -15,16 +16,14 @@ struct Fixture Config _config; Fixture() : _config() - { - } + { } Fixture(BasicType bt, CollectionType ct = CollectionType::SINGLE, bool fastSearch_ = false, bool huge_ = false) : _config(bt, ct, fastSearch_, huge_) - { - } + { } }; TEST_F("test default attribute config", Fixture) @@ -110,5 +109,24 @@ TEST("Test GrowStrategy consistency") { EXPECT_EQUAL(0.4f, g.getMultiValueAllocGrowFactor()); } +TEST("DictionaryConfig") { + using Ordering = DictionaryConfig::Ordering; + EXPECT_EQUAL(Ordering::ORDERED, DictionaryConfig().getOrdering()); + EXPECT_EQUAL(Ordering::ORDERED, DictionaryConfig(Ordering::ORDERED).getOrdering()); + EXPECT_EQUAL(Ordering::UNORDERED, DictionaryConfig(Ordering::UNORDERED).getOrdering()); + EXPECT_EQUAL(DictionaryConfig(Ordering::ORDERED), DictionaryConfig(Ordering::ORDERED)); + EXPECT_EQUAL(DictionaryConfig(Ordering::UNORDERED), DictionaryConfig(Ordering::UNORDERED)); + EXPECT_NOT_EQUAL(DictionaryConfig(Ordering::UNORDERED), DictionaryConfig(Ordering::ORDERED)); + EXPECT_NOT_EQUAL(DictionaryConfig(Ordering::ORDERED), DictionaryConfig(Ordering::UNORDERED)); + EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) == + Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED))); + EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) == + Config().set_dictionary_config(DictionaryConfig(Ordering::ORDERED))); + EXPECT_FALSE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) != + Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED))); + EXPECT_TRUE(Config().set_dictionary_config(DictionaryConfig(Ordering::UNORDERED)) != + Config().set_dictionary_config(DictionaryConfig(Ordering::ORDERED))); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index 7752baa603a..065e9c14de9 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -4,7 +4,7 @@ namespace search::attribute { -Config::Config() : +Config::Config() noexcept : _basicType(BasicType::NONE), _type(CollectionType::SINGLE), _fastSearch(false), @@ -14,6 +14,7 @@ Config::Config() : _isFilter(false), _fastAccess(false), _mutable(false), + _dictionary(), _growStrategy(), _compactionStrategy(), _predicateParams(), @@ -23,7 +24,7 @@ Config::Config() : { } -Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) +Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) noexcept : _basicType(bt), _type(ct), _fastSearch(fastSearch_), @@ -33,6 +34,7 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) _isFilter(false), _fastAccess(false), _mutable(false), + _dictionary(), _growStrategy(), _compactionStrategy(), _predicateParams(), @@ -60,6 +62,7 @@ Config::operator==(const Config &b) const _isFilter == b._isFilter && _fastAccess == b._fastAccess && _mutable == b._mutable && + _dictionary == b._dictionary && _growStrategy == b._growStrategy && _compactionStrategy == b._compactionStrategy && _predicateParams == b._predicateParams && diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 8df7f29590b..c1b30303606 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -8,6 +8,7 @@ #include "predicate_params.h" #include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchcommon/common/growstrategy.h> +#include <vespa/searchcommon/common/dictionary_config.h> #include <vespa/eval/eval/value_type.h> #include <cassert> #include <optional> @@ -21,9 +22,13 @@ namespace search::attribute { */ class Config { public: - Config(); - Config(BasicType bt, CollectionType ct = CollectionType::SINGLE, - bool fastSearch_ = false, bool huge_ = false); + Config() noexcept; + Config(BasicType bt) noexcept : Config(bt, CollectionType::SINGLE) { } + Config(BasicType bt, CollectionType ct) noexcept : Config(bt, ct, false) { } + Config(BasicType bt, CollectionType ct, bool fastSearch_) noexcept + : Config(bt, ct, fastSearch_, false) + {} + Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) noexcept; Config(const Config &); Config & operator = (const Config &); Config(Config &&) noexcept; @@ -62,6 +67,7 @@ public: const GrowStrategy & getGrowStrategy() const { return _growStrategy; } const CompactionStrategy &getCompactionStrategy() const { return _compactionStrategy; } + const DictionaryConfig & get_dictionary_config() const { return _dictionary; } Config & setHuge(bool v) { _huge = v; return *this;} Config & setFastSearch(bool v) { _fastSearch = v; return *this; } Config & setPredicateParams(const PredicateParams &v) { _predicateParams = v; return *this; } @@ -107,11 +113,14 @@ public: * Hide weight information when searching in attributes. */ Config & setIsFilter(bool isFilter) { _isFilter = isFilter; return *this; } - Config & setMutable(bool isMutable) { _mutable = isMutable; return *this; } Config & setFastAccess(bool v) { _fastAccess = v; return *this; } Config & setGrowStrategy(const GrowStrategy &gs) { _growStrategy = gs; return *this; } - Config &setCompactionStrategy(const CompactionStrategy &compactionStrategy) { _compactionStrategy = compactionStrategy; return *this; } + Config & setCompactionStrategy(const CompactionStrategy &compactionStrategy) { + _compactionStrategy = compactionStrategy; + return *this; + } + Config & set_dictionary_config(const DictionaryConfig & cfg) { _dictionary = cfg; return *this; } bool operator!=(const Config &b) const { return !(operator==(b)); } bool operator==(const Config &b) const; @@ -125,11 +134,12 @@ private: bool _isFilter; bool _fastAccess; bool _mutable; - GrowStrategy _growStrategy; - CompactionStrategy _compactionStrategy; - PredicateParams _predicateParams; - vespalib::eval::ValueType _tensorType; - DistanceMetric _distance_metric; + DictionaryConfig _dictionary; + GrowStrategy _growStrategy; + CompactionStrategy _compactionStrategy; + PredicateParams _predicateParams; + vespalib::eval::ValueType _tensorType; + DistanceMetric _distance_metric; std::optional<HnswIndexParams> _hnsw_index_params; }; diff --git a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt index bf5686280e6..46824bb9750 100644 --- a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(searchcommon_searchcommon_common OBJECT SOURCES compaction_strategy.cpp datatype.cpp + dictionary_config.cpp growstrategy.cpp schema.cpp schemaconfigurer.cpp diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h index ae354a4c4d2..8a9b94f3af8 100644 --- a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h +++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h @@ -2,7 +2,6 @@ #pragma once -#include <stdint.h> #include <iosfwd> namespace search { @@ -16,12 +15,12 @@ private: double _maxDeadBytesRatio; // Max ratio of dead bytes before compaction double _maxDeadAddressSpaceRatio; // Max ratio of dead address space before compaction public: - CompactionStrategy() + CompactionStrategy() noexcept : _maxDeadBytesRatio(0.2), _maxDeadAddressSpaceRatio(0.2) { } - CompactionStrategy(double maxDeadBytesRatio, double maxDeadAddressSpaceRatio) + CompactionStrategy(double maxDeadBytesRatio, double maxDeadAddressSpaceRatio) noexcept : _maxDeadBytesRatio(maxDeadBytesRatio), _maxDeadAddressSpaceRatio(maxDeadAddressSpaceRatio) { diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp b/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp new file mode 100644 index 00000000000..00b6ae2710f --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp @@ -0,0 +1,20 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dictionary_config.h" +#include <ostream> +#include <cassert> + +namespace search { + +std::ostream& +operator<<(std::ostream& os, const DictionaryConfig & cfg) { + switch(cfg.getOrdering()) { + case DictionaryConfig::Ordering::ORDERED: + return os << "ORDERED"; + case DictionaryConfig::Ordering::UNORDERED: + return os << "UNORDERED"; + } + assert(false); +} + +} // namespace search diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h b/searchcommon/src/vespa/searchcommon/common/dictionary_config.h new file mode 100644 index 00000000000..c6c99a137e4 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/dictionary_config.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <iosfwd> + +namespace search { + +class DictionaryConfig { +public: + enum class Ordering { ORDERED, UNORDERED }; + DictionaryConfig() noexcept : _ordering(Ordering::ORDERED) {} + DictionaryConfig(Ordering ordering) noexcept : _ordering(ordering) {} + Ordering getOrdering() const { return _ordering; } + bool operator == (const DictionaryConfig & b) const { return _ordering == b._ordering; } +private: + Ordering _ordering; +}; + +std::ostream& operator<<(std::ostream& os, const DictionaryConfig & cfg); + +} // namespace search diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.h b/searchcommon/src/vespa/searchcommon/common/growstrategy.h index f7c5c030d95..60ca0977ea9 100644 --- a/searchcommon/src/vespa/searchcommon/common/growstrategy.h +++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.h @@ -16,11 +16,11 @@ private: uint32_t _docsGrowDelta; float _multiValueAllocGrowFactor; public: - GrowStrategy() + GrowStrategy() noexcept : GrowStrategy(1024, 0.5, 0, 0.2) {} GrowStrategy(uint32_t docsInitialCapacity, float docsGrowFactor, - uint32_t docsGrowDelta, float multiValueAllocGrowFactor) + uint32_t docsGrowDelta, float multiValueAllocGrowFactor) noexcept : _docsInitialCapacity(docsInitialCapacity), _docsGrowFactor(docsGrowFactor), _docsGrowDelta(docsGrowDelta), diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 991f1f03ee7..145a021801d 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -50,6 +50,22 @@ getCollectionTypeMap() static DataTypeMap _dataTypeMap = getDataTypeMap(); static CollectionTypeMap _collectionTypeMap = getCollectionTypeMap(); +DictionaryConfig::Ordering +convert(AttributesConfig::Attribute::Dictionary::Ordering ordering_cfg) { + switch (ordering_cfg) { + case AttributesConfig::Attribute::Dictionary::Ordering::ORDERED: + return DictionaryConfig::Ordering::ORDERED; + case AttributesConfig::Attribute::Dictionary::Ordering::UNORDERED: + return DictionaryConfig::Ordering::UNORDERED; + } + assert(false); +} + +DictionaryConfig +convert_dictionary(const AttributesConfig::Attribute::Dictionary & dictionary) { + return DictionaryConfig(convert(dictionary.ordering)); +} + } namespace search::attribute { @@ -74,6 +90,7 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) predicateParams.setBounds(cfg.lowerbound, cfg.upperbound); predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold); retval.setPredicateParams(predicateParams); + retval.set_dictionary_config(convert_dictionary(cfg.dictionary)); using CfgDm = AttributesConfig::Attribute::Distancemetric; DistanceMetric dm(DistanceMetric::Euclidean); switch (cfg.distancemetric) { |