diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-07-05 14:39:37 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-07-05 14:39:37 +0000 |
commit | 3182d80c0dbccfe2f2a854a9eaf3e4308918b61a (patch) | |
tree | 75bdf6a589dd549dfc7948275a72195fc5533e38 /searchlib/src/tests/attribute | |
parent | a52700972fad8fc8b98d7eafc41086499a8fbd56 (diff) |
- Consolidate on isFilter.
- Add has_weight_iterator to IDocumentWeightAttribute to allow fallback to bitvector.
- Allow filter attributes to enjoy IDirectWeightedSet optimization.
Diffstat (limited to 'searchlib/src/tests/attribute')
4 files changed, 61 insertions, 129 deletions
diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index 6e622c840b6..5fa8889a01d 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -53,15 +53,9 @@ struct BitVectorTest StringAttribute & asString(AttributePtr &v); FloatingPointAttribute & asFloat(AttributePtr &v); - AttributePtr - make(Config cfg, - const vespalib::string &pref, - bool fastSearch, - bool enableOnlyBitVector, - bool filter); + AttributePtr make(Config cfg, const vespalib::string &pref, bool fastSearch, bool filter); - void - addDocs(const AttributePtr &v, size_t sz); + void addDocs(const AttributePtr &v, size_t sz); template <typename VectorType> void populate(VectorType &v, uint32_t low, uint32_t high, bool set); @@ -69,22 +63,16 @@ struct BitVectorTest template <typename VectorType> void populateAll(VectorType &v, uint32_t low, uint32_t high, bool set); - void - buildTermQuery(std::vector<char> & buffer, - const vespalib::string & index, - const vespalib::string & term, bool prefix); + void buildTermQuery(std::vector<char> & buffer, const vespalib::string & index, const vespalib::string & term, bool prefix); template <typename V> - vespalib::string - getSearchStr(); + vespalib::string getSearchStr(); template <typename V, typename T> - SearchContextPtr - getSearch(const V & vec, const T & term, bool prefix, bool useBitVector); + SearchContextPtr getSearch(const V & vec, const T & term, bool prefix, bool useBitVector); template <typename V> - SearchContextPtr - getSearch(const V & vec, bool useBitVector); + SearchContextPtr getSearch(const V & vec, bool useBitVector); void checkSearch(AttributePtr v, @@ -107,10 +95,7 @@ struct BitVectorTest template <typename VectorType, typename BufferType> void - test(BasicType bt, CollectionType ct, const vespalib::string &pref, - bool fastSearch, - bool enableOnlyBitVector, - bool filter); + test(BasicType bt, CollectionType ct, const vespalib::string &pref, bool fastSearch, bool filter); template <typename VectorType, typename BufferType> void @@ -195,8 +180,7 @@ BitVectorTest::getSearchStr<StringAttribute>() template <typename V, typename T> SearchContextPtr -BitVectorTest::getSearch(const V &vec, const T &term, bool prefix, - bool useBitVector) +BitVectorTest::getSearch(const V &vec, const T &term, bool prefix, bool useBitVector) { std::vector<char> query; vespalib::asciistream ss; @@ -211,8 +195,7 @@ BitVectorTest::getSearch(const V &vec, const T &term, bool prefix, template <> SearchContextPtr -BitVectorTest::getSearch<IntegerAttribute>(const IntegerAttribute &v, - bool useBitVector) +BitVectorTest::getSearch<IntegerAttribute>(const IntegerAttribute &v, bool useBitVector) { return getSearch<IntegerAttribute>(v, "[-42;-42]", false, useBitVector); } @@ -220,32 +203,23 @@ BitVectorTest::getSearch<IntegerAttribute>(const IntegerAttribute &v, template <> SearchContextPtr BitVectorTest:: -getSearch<FloatingPointAttribute>(const FloatingPointAttribute &v, - bool useBitVector) +getSearch<FloatingPointAttribute>(const FloatingPointAttribute &v, bool useBitVector) { - return getSearch<FloatingPointAttribute>(v, "[-42.0;-42.0]", false, - useBitVector); + return getSearch<FloatingPointAttribute>(v, "[-42.0;-42.0]", false, useBitVector); } template <> SearchContextPtr -BitVectorTest::getSearch<StringAttribute>(const StringAttribute &v, - bool useBitVector) +BitVectorTest::getSearch<StringAttribute>(const StringAttribute &v, bool useBitVector) { - return getSearch<StringAttribute, const vespalib::string &> - (v, "foo", false, useBitVector); + return getSearch<StringAttribute, const vespalib::string &>(v, "foo", false, useBitVector); } BitVectorTest::AttributePtr -BitVectorTest::make(Config cfg, - const vespalib::string &pref, - bool fastSearch, - bool enableOnlyBitVector, - bool filter) +BitVectorTest::make(Config cfg, const vespalib::string &pref, bool fastSearch, bool filter) { cfg.setFastSearch(fastSearch); - cfg.setEnableOnlyBitVector(enableOnlyBitVector); cfg.setIsFilter(filter); AttributePtr v = AttributeFactory::createAttribute(pref, cfg); return v; @@ -267,11 +241,9 @@ BitVectorTest::addDocs(const AttributePtr &v, size_t sz) template <> void -BitVectorTest::populate(IntegerAttribute &v, - uint32_t low, uint32_t high, - bool set) +BitVectorTest::populate(IntegerAttribute &v, uint32_t low, uint32_t high, bool set) { - for(size_t i(low), m(high); i < m; i+= 5) { + for (size_t i(low), m(high); i < m; i+= 5) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -288,11 +260,9 @@ BitVectorTest::populate(IntegerAttribute &v, template <> void -BitVectorTest::populate(FloatingPointAttribute &v, - uint32_t low, uint32_t high, - bool set) +BitVectorTest::populate(FloatingPointAttribute &v, uint32_t low, uint32_t high, bool set) { - for(size_t i(low), m(high); i < m; i+= 5) { + for (size_t i(low), m(high); i < m; i+= 5) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -309,11 +279,9 @@ BitVectorTest::populate(FloatingPointAttribute &v, template <> void -BitVectorTest::populate(StringAttribute &v, - uint32_t low, uint32_t high, - bool set) +BitVectorTest::populate(StringAttribute &v, uint32_t low, uint32_t high, bool set) { - for(size_t i(low), m(high); i < m; i+= 5) { + for (size_t i(low), m(high); i < m; i+= 5) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -329,11 +297,9 @@ BitVectorTest::populate(StringAttribute &v, template <> void -BitVectorTest::populateAll(IntegerAttribute &v, - uint32_t low, uint32_t high, - bool set) +BitVectorTest::populateAll(IntegerAttribute &v, uint32_t low, uint32_t high, bool set) { - for(size_t i(low), m(high); i < m; ++i) { + for (size_t i(low), m(high); i < m; ++i) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -351,11 +317,9 @@ BitVectorTest::populateAll(IntegerAttribute &v, template <> void -BitVectorTest::populateAll(FloatingPointAttribute &v, - uint32_t low, uint32_t high, - bool set) +BitVectorTest::populateAll(FloatingPointAttribute &v, uint32_t low, uint32_t high, bool set) { - for(size_t i(low), m(high); i < m; ++i) { + for (size_t i(low), m(high); i < m; ++i) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -373,11 +337,9 @@ BitVectorTest::populateAll(FloatingPointAttribute &v, template <> void -BitVectorTest::populateAll(StringAttribute &v, - uint32_t low, uint32_t high, - bool set) +BitVectorTest::populateAll(StringAttribute &v, uint32_t low, uint32_t high, bool set) { - for(size_t i(low), m(high); i < m; ++i) { + for (size_t i(low), m(high); i < m; ++i) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -416,8 +378,7 @@ BitVectorTest::checkSearch(AttributePtr v, assert(!checkStride || (docId % 5) == 2u); sb->unpack(docId); EXPECT_EQUAL(md.getDocId(), docId); - if (v->getCollectionType() == CollectionType::SINGLE || - !weights) { + if (v->getCollectionType() == CollectionType::SINGLE || !weights) { EXPECT_EQUAL(1, md.getWeight()); } else if (v->getCollectionType() == CollectionType::ARRAY) { EXPECT_EQUAL(2, md.getWeight()); @@ -456,15 +417,10 @@ BitVectorTest::checkSearch(AttributePtr v, template <typename VectorType, typename BufferType> void -BitVectorTest::test(BasicType bt, - CollectionType ct, - const vespalib::string &pref, - bool fastSearch, - bool enableOnlyBitVector, - bool filter) +BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pref, bool fastSearch, bool filter) { Config cfg(bt, ct); - AttributePtr v = make(cfg, pref, fastSearch, enableOnlyBitVector, filter); + AttributePtr v = make(cfg, pref, fastSearch, filter); addDocs(v, 1024); auto &tv = as<VectorType>(v); populate(tv, 2, 1023, true); @@ -472,7 +428,7 @@ BitVectorTest::test(BasicType bt, SearchContextPtr sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); sc = getSearch<VectorType>(tv, false); - checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && !filter, true); + checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); const search::IDocumentWeightAttribute *dwa = v->asDocumentWeightAttribute(); if (dwa != nullptr) { search::IDocumentWeightAttribute::LookupResult lres = @@ -481,8 +437,8 @@ BitVectorTest::test(BasicType bt, using SI = search::queryeval::SearchIterator; TermFieldMatchData md; SI::UP dwsi(new DWSI(md, *dwa, lres)); - if (!enableOnlyBitVector) { - checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true); + if (!filter) { + TEST_DO(checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true)); } else { dwsi->initRange(1, v->getCommittedDocIdLimit()); EXPECT_TRUE(dwsi->isAtEnd()); @@ -490,13 +446,13 @@ BitVectorTest::test(BasicType bt, } populate(tv, 2, 973, false); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 977, 1022, 10, !enableOnlyBitVector &&!filter, true); + checkSearch(v, std::move(sc), 977, 1022, 10, !filter, true); populate(tv, 2, 973, true); sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); addDocs(v, 15000); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && !filter, true); + checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); populateAll(tv, 10, 15000, true); sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 14999, 14992, !fastSearch && !filter, false); @@ -508,85 +464,65 @@ void BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pref) { LOG(info, "test run, pref is %s", pref.c_str()); - test<VectorType, BufferType>(bt, ct, pref, false, false, false); - test<VectorType, BufferType>(bt, ct, pref, false, false, true); - test<VectorType, BufferType>(bt, ct, pref, true, false, false); - test<VectorType, BufferType>(bt, ct, pref, true, false, true); - test<VectorType, BufferType>(bt, ct, pref, true, true, false); - test<VectorType, BufferType>(bt, ct, pref, true, true, true); + test<VectorType, BufferType>(bt, ct, pref, false, false); + test<VectorType, BufferType>(bt, ct, pref, false, true); + test<VectorType, BufferType>(bt, ct, pref, true, false); + test<VectorType, BufferType>(bt, ct, pref, true, true); } TEST_F("Test bitvectors with single value int32", BitVectorTest) { f.template test<IntegerAttribute, - IntegerAttribute::largeint_t>(BasicType::INT32, - CollectionType::SINGLE, - "int32_sv"); + IntegerAttribute::largeint_t>(BasicType::INT32, CollectionType::SINGLE, "int32_sv"); } TEST_F("Test bitvectors with array value int32", BitVectorTest) { f.template test<IntegerAttribute, - IntegerAttribute::largeint_t>(BasicType::INT32, - CollectionType::ARRAY, - "int32_a"); + IntegerAttribute::largeint_t>(BasicType::INT32, CollectionType::ARRAY, "int32_a"); } TEST_F("Test bitvectors with weighted set value int32", BitVectorTest) { f.template test<IntegerAttribute, - IntegerAttribute::WeightedInt>(BasicType::INT32, - CollectionType::WSET, - "int32_sv"); + IntegerAttribute::WeightedInt>(BasicType::INT32, CollectionType::WSET, "int32_sv"); } TEST_F("Test bitvectors with single value double", BitVectorTest) { f.template test<FloatingPointAttribute, - double>(BasicType::DOUBLE, - CollectionType::SINGLE, - "double_sv"); + double>(BasicType::DOUBLE, CollectionType::SINGLE, "double_sv"); } TEST_F("Test bitvectors with array value double", BitVectorTest) { f.template test<FloatingPointAttribute, - double>(BasicType::DOUBLE, - CollectionType::ARRAY, - "double_a"); + double>(BasicType::DOUBLE, CollectionType::ARRAY, "double_a"); } TEST_F("Test bitvectors with weighted set value double", BitVectorTest) { f.template test<FloatingPointAttribute, - FloatingPointAttribute::WeightedFloat>(BasicType::DOUBLE, - CollectionType::WSET, - "double_ws"); + FloatingPointAttribute::WeightedFloat>(BasicType::DOUBLE, CollectionType::WSET, "double_ws"); } TEST_F("Test bitvectors with single value string", BitVectorTest) { f.template test<StringAttribute, - vespalib::string>(BasicType::STRING, - CollectionType::SINGLE, - "string_sv"); + vespalib::string>(BasicType::STRING, CollectionType::SINGLE, "string_sv"); } TEST_F("Test bitvectors with array value string", BitVectorTest) { f.template test<StringAttribute, - vespalib::string>(BasicType::STRING, - CollectionType::ARRAY, - "string_a"); + vespalib::string>(BasicType::STRING, CollectionType::ARRAY, "string_a"); } TEST_F("Test bitvectors with weighted set value string", BitVectorTest) { f.template test<StringAttribute, - StringAttribute::WeightedString>(BasicType::STRING, - CollectionType::WSET, - "string_ws"); + StringAttribute::WeightedString>(BasicType::STRING, CollectionType::WSET, "string_ws"); } @@ -633,5 +569,4 @@ TEST("Test that bitvector iterators adheres to SearchIterator requirements") { } } - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index 57029f92111..227dbfadbc0 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -4,9 +4,7 @@ #include <vespa/searchcommon/attribute/status.h> #include <vespa/searchlib/attribute/postingstore.h> #include <vespa/searchlib/attribute/enumstore.hpp> -#include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreerootbase.hpp> -#include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/searchlib/attribute/postingstore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/gtest/gtest.h> @@ -42,7 +40,7 @@ std::ostream& operator<<(std::ostream& os, const PostingStoreSetup setup) Config make_config(PostingStoreSetup param) { Config cfg; - cfg.setEnableOnlyBitVector(param.enable_only_bitvector); + cfg.setIsFilter(param.enable_only_bitvector); return cfg; } @@ -212,8 +210,7 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1)); EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2)); auto usage_after = store.getMemoryUsage(); - if (sequence_length < huge_sequence_length || - !_config.getEnableOnlyBitVector()) { + if ((sequence_length < huge_sequence_length) || !_config.getIsFilter()) { EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes()); } else { EXPECT_EQ(usage_before.deadBytes(), usage_after.deadBytes()); diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index 00e2a82d24e..25de1105973 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -471,7 +471,7 @@ PostingListAttributeTest::checkPostingList(const VectorType & vec, const std::ve } EXPECT_EQ(doc, docEnd); } else { - EXPECT_TRUE(has_bitvector && vec.getEnableOnlyBitVector()); + EXPECT_TRUE(has_bitvector && vec.getIsFilter()); numHits = postingList.getBitVectorEntry(find_result.second)->_bv->reader().countTrueBits(); } if (has_bitvector) { @@ -612,21 +612,21 @@ PostingListAttributeTest::testPostingList(bool enable_only_bitvector, uint32_t n { Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("sint32", cfg); testPostingList<Int32PostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::ARRAY)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("aint32", cfg); testPostingList<Int32ArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::WSET)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("wsint32", cfg); testPostingList<Int32WsetPostingListAttribute>(ptr1, numDocs, values); } @@ -640,21 +640,21 @@ PostingListAttributeTest::testPostingList(bool enable_only_bitvector, uint32_t n { Config cfg(Config(BasicType::FLOAT, CollectionType::SINGLE)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("sfloat", cfg); testPostingList<FloatPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::ARRAY)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("afloat", cfg); testPostingList<FloatArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::WSET)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("wsfloat", cfg); testPostingList<FloatWsetPostingListAttribute>(ptr1, numDocs, values); } @@ -674,21 +674,21 @@ PostingListAttributeTest::testPostingList(bool enable_only_bitvector, uint32_t n { Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("sstr", cfg); testPostingList<StringPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::ARRAY)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("astr", cfg); testPostingList<StringArrayPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::WSET)); cfg.setFastSearch(true); - cfg.setEnableOnlyBitVector(enable_only_bitvector); + cfg.setIsFilter(enable_only_bitvector); AttributePtr ptr1 = create_attribute("wsstr", cfg); testPostingList<StringWsetPostingListAttribute>(ptr1, numDocs, charValues); } diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 2f3684874ee..9cfd5946dbb 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -510,8 +510,8 @@ TEST("require that single weighted set turns filter on filter fields") { SimpleStringTerm node("foo", "", 0, Weight(1)); Result result = do_search(attribute_manager, node, strict); EXPECT_EQUAL(3u, result.est_hits); - EXPECT_TRUE(result.iterator_dump.find("DocumentWeightSearchIterator") == vespalib::string::npos); - EXPECT_TRUE(result.iterator_dump.find("FilterAttributePostingListIteratorT") != vespalib::string::npos); + EXPECT_TRUE(result.iterator_dump.find("DocumentWeightSearchIterator") != vespalib::string::npos); + EXPECT_TRUE(result.iterator_dump.find("FilterAttributePostingListIteratorT") == vespalib::string::npos); ASSERT_EQUAL(3u, result.hits.size()); EXPECT_FALSE(result.est_empty); EXPECT_EQUAL(20u, result.hits[0].docid); |