diff options
Diffstat (limited to 'searchlib')
50 files changed, 524 insertions, 524 deletions
diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index c072f722677..82c1839e63b 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -10,7 +10,7 @@ #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/fastos/file.h> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP("attributefilewriter_test"); @@ -24,7 +24,7 @@ namespace { vespalib::string testFileName("test.dat"); vespalib::string hello("Hello world"); -void removeTestFile() { FastOS_File::Delete(testFileName.c_str()); } +void removeTestFile() { std::filesystem::remove(std::filesystem::path(testFileName)); } struct Fixture { TuneFileAttributes _tuneFileAttributes; diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index 5fa8889a01d..6e622c840b6 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -53,9 +53,15 @@ struct BitVectorTest StringAttribute & asString(AttributePtr &v); FloatingPointAttribute & asFloat(AttributePtr &v); - AttributePtr make(Config cfg, const vespalib::string &pref, bool fastSearch, bool filter); + AttributePtr + make(Config cfg, + const vespalib::string &pref, + bool fastSearch, + bool enableOnlyBitVector, + bool filter); - void addDocs(const AttributePtr &v, size_t sz); + void + addDocs(const AttributePtr &v, size_t sz); template <typename VectorType> void populate(VectorType &v, uint32_t low, uint32_t high, bool set); @@ -63,16 +69,22 @@ struct BitVectorTest template <typename VectorType> void populateAll(VectorType &v, uint32_t low, uint32_t high, bool set); - void buildTermQuery(std::vector<char> & buffer, const vespalib::string & index, const vespalib::string & term, bool prefix); + void + buildTermQuery(std::vector<char> & buffer, + const vespalib::string & index, + const vespalib::string & term, bool prefix); template <typename V> - vespalib::string getSearchStr(); + vespalib::string + getSearchStr(); template <typename V, typename T> - SearchContextPtr getSearch(const V & vec, const T & term, bool prefix, bool useBitVector); + SearchContextPtr + getSearch(const V & vec, const T & term, bool prefix, bool useBitVector); template <typename V> - SearchContextPtr getSearch(const V & vec, bool useBitVector); + SearchContextPtr + getSearch(const V & vec, bool useBitVector); void checkSearch(AttributePtr v, @@ -95,7 +107,10 @@ struct BitVectorTest template <typename VectorType, typename BufferType> void - test(BasicType bt, CollectionType ct, const vespalib::string &pref, bool fastSearch, bool filter); + test(BasicType bt, CollectionType ct, const vespalib::string &pref, + bool fastSearch, + bool enableOnlyBitVector, + bool filter); template <typename VectorType, typename BufferType> void @@ -180,7 +195,8 @@ BitVectorTest::getSearchStr<StringAttribute>() template <typename V, typename T> SearchContextPtr -BitVectorTest::getSearch(const V &vec, const T &term, bool prefix, bool useBitVector) +BitVectorTest::getSearch(const V &vec, const T &term, bool prefix, + bool useBitVector) { std::vector<char> query; vespalib::asciistream ss; @@ -195,7 +211,8 @@ BitVectorTest::getSearch(const V &vec, const T &term, bool prefix, bool useBitVe template <> SearchContextPtr -BitVectorTest::getSearch<IntegerAttribute>(const IntegerAttribute &v, bool useBitVector) +BitVectorTest::getSearch<IntegerAttribute>(const IntegerAttribute &v, + bool useBitVector) { return getSearch<IntegerAttribute>(v, "[-42;-42]", false, useBitVector); } @@ -203,23 +220,32 @@ BitVectorTest::getSearch<IntegerAttribute>(const IntegerAttribute &v, bool useBi template <> SearchContextPtr BitVectorTest:: -getSearch<FloatingPointAttribute>(const FloatingPointAttribute &v, bool useBitVector) +getSearch<FloatingPointAttribute>(const FloatingPointAttribute &v, + bool useBitVector) { - return getSearch<FloatingPointAttribute>(v, "[-42.0;-42.0]", false, useBitVector); + return getSearch<FloatingPointAttribute>(v, "[-42.0;-42.0]", false, + useBitVector); } template <> SearchContextPtr -BitVectorTest::getSearch<StringAttribute>(const StringAttribute &v, bool useBitVector) +BitVectorTest::getSearch<StringAttribute>(const StringAttribute &v, + bool useBitVector) { - return getSearch<StringAttribute, const vespalib::string &>(v, "foo", false, useBitVector); + return getSearch<StringAttribute, const vespalib::string &> + (v, "foo", false, useBitVector); } BitVectorTest::AttributePtr -BitVectorTest::make(Config cfg, const vespalib::string &pref, bool fastSearch, bool filter) +BitVectorTest::make(Config cfg, + const vespalib::string &pref, + bool fastSearch, + bool enableOnlyBitVector, + bool filter) { cfg.setFastSearch(fastSearch); + cfg.setEnableOnlyBitVector(enableOnlyBitVector); cfg.setIsFilter(filter); AttributePtr v = AttributeFactory::createAttribute(pref, cfg); return v; @@ -241,9 +267,11 @@ BitVectorTest::addDocs(const AttributePtr &v, size_t sz) template <> void -BitVectorTest::populate(IntegerAttribute &v, uint32_t low, uint32_t high, bool set) +BitVectorTest::populate(IntegerAttribute &v, + uint32_t low, uint32_t high, + bool set) { - for (size_t i(low), m(high); i < m; i+= 5) { + for(size_t i(low), m(high); i < m; i+= 5) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -260,9 +288,11 @@ BitVectorTest::populate(IntegerAttribute &v, uint32_t low, uint32_t high, bool s template <> void -BitVectorTest::populate(FloatingPointAttribute &v, uint32_t low, uint32_t high, bool set) +BitVectorTest::populate(FloatingPointAttribute &v, + uint32_t low, uint32_t high, + bool set) { - for (size_t i(low), m(high); i < m; i+= 5) { + for(size_t i(low), m(high); i < m; i+= 5) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -279,9 +309,11 @@ BitVectorTest::populate(FloatingPointAttribute &v, uint32_t low, uint32_t high, template <> void -BitVectorTest::populate(StringAttribute &v, uint32_t low, uint32_t high, bool set) +BitVectorTest::populate(StringAttribute &v, + uint32_t low, uint32_t high, + bool set) { - for (size_t i(low), m(high); i < m; i+= 5) { + for(size_t i(low), m(high); i < m; i+= 5) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -297,9 +329,11 @@ BitVectorTest::populate(StringAttribute &v, uint32_t low, uint32_t high, bool se template <> void -BitVectorTest::populateAll(IntegerAttribute &v, uint32_t low, uint32_t high, bool set) +BitVectorTest::populateAll(IntegerAttribute &v, + uint32_t low, uint32_t high, + bool set) { - for (size_t i(low), m(high); i < m; ++i) { + for(size_t i(low), m(high); i < m; ++i) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -317,9 +351,11 @@ BitVectorTest::populateAll(IntegerAttribute &v, uint32_t low, uint32_t high, boo template <> void -BitVectorTest::populateAll(FloatingPointAttribute &v, uint32_t low, uint32_t high, bool set) +BitVectorTest::populateAll(FloatingPointAttribute &v, + uint32_t low, uint32_t high, + bool set) { - for (size_t i(low), m(high); i < m; ++i) { + for(size_t i(low), m(high); i < m; ++i) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -337,9 +373,11 @@ BitVectorTest::populateAll(FloatingPointAttribute &v, uint32_t low, uint32_t hig template <> void -BitVectorTest::populateAll(StringAttribute &v, uint32_t low, uint32_t high, bool set) +BitVectorTest::populateAll(StringAttribute &v, + uint32_t low, uint32_t high, + bool set) { - for (size_t i(low), m(high); i < m; ++i) { + for(size_t i(low), m(high); i < m; ++i) { if (!set) { v.clearDoc(i); } else if (v.hasMultiValue()) { @@ -378,7 +416,8 @@ BitVectorTest::checkSearch(AttributePtr v, assert(!checkStride || (docId % 5) == 2u); sb->unpack(docId); EXPECT_EQUAL(md.getDocId(), docId); - if (v->getCollectionType() == CollectionType::SINGLE || !weights) { + if (v->getCollectionType() == CollectionType::SINGLE || + !weights) { EXPECT_EQUAL(1, md.getWeight()); } else if (v->getCollectionType() == CollectionType::ARRAY) { EXPECT_EQUAL(2, md.getWeight()); @@ -417,10 +456,15 @@ BitVectorTest::checkSearch(AttributePtr v, template <typename VectorType, typename BufferType> void -BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pref, bool fastSearch, bool filter) +BitVectorTest::test(BasicType bt, + CollectionType ct, + const vespalib::string &pref, + bool fastSearch, + bool enableOnlyBitVector, + bool filter) { Config cfg(bt, ct); - AttributePtr v = make(cfg, pref, fastSearch, filter); + AttributePtr v = make(cfg, pref, fastSearch, enableOnlyBitVector, filter); addDocs(v, 1024); auto &tv = as<VectorType>(v); populate(tv, 2, 1023, true); @@ -428,7 +472,7 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre SearchContextPtr sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); sc = getSearch<VectorType>(tv, false); - checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); + checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && !filter, true); const search::IDocumentWeightAttribute *dwa = v->asDocumentWeightAttribute(); if (dwa != nullptr) { search::IDocumentWeightAttribute::LookupResult lres = @@ -437,8 +481,8 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre using SI = search::queryeval::SearchIterator; TermFieldMatchData md; SI::UP dwsi(new DWSI(md, *dwa, lres)); - if (!filter) { - TEST_DO(checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true)); + if (!enableOnlyBitVector) { + checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true); } else { dwsi->initRange(1, v->getCommittedDocIdLimit()); EXPECT_TRUE(dwsi->isAtEnd()); @@ -446,13 +490,13 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre } populate(tv, 2, 973, false); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 977, 1022, 10, !filter, true); + checkSearch(v, std::move(sc), 977, 1022, 10, !enableOnlyBitVector &&!filter, true); populate(tv, 2, 973, true); sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); addDocs(v, 15000); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); + checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && !filter, true); populateAll(tv, 10, 15000, true); sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 14999, 14992, !fastSearch && !filter, false); @@ -464,65 +508,85 @@ void BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pref) { LOG(info, "test run, pref is %s", pref.c_str()); - test<VectorType, BufferType>(bt, ct, pref, false, false); - test<VectorType, BufferType>(bt, ct, pref, false, true); - test<VectorType, BufferType>(bt, ct, pref, true, false); - test<VectorType, BufferType>(bt, ct, pref, true, true); + test<VectorType, BufferType>(bt, ct, pref, false, false, false); + test<VectorType, BufferType>(bt, ct, pref, false, false, true); + test<VectorType, BufferType>(bt, ct, pref, true, false, false); + test<VectorType, BufferType>(bt, ct, pref, true, false, true); + test<VectorType, BufferType>(bt, ct, pref, true, true, false); + test<VectorType, BufferType>(bt, ct, pref, true, true, true); } TEST_F("Test bitvectors with single value int32", BitVectorTest) { f.template test<IntegerAttribute, - IntegerAttribute::largeint_t>(BasicType::INT32, CollectionType::SINGLE, "int32_sv"); + IntegerAttribute::largeint_t>(BasicType::INT32, + CollectionType::SINGLE, + "int32_sv"); } TEST_F("Test bitvectors with array value int32", BitVectorTest) { f.template test<IntegerAttribute, - IntegerAttribute::largeint_t>(BasicType::INT32, CollectionType::ARRAY, "int32_a"); + IntegerAttribute::largeint_t>(BasicType::INT32, + CollectionType::ARRAY, + "int32_a"); } TEST_F("Test bitvectors with weighted set value int32", BitVectorTest) { f.template test<IntegerAttribute, - IntegerAttribute::WeightedInt>(BasicType::INT32, CollectionType::WSET, "int32_sv"); + IntegerAttribute::WeightedInt>(BasicType::INT32, + CollectionType::WSET, + "int32_sv"); } TEST_F("Test bitvectors with single value double", BitVectorTest) { f.template test<FloatingPointAttribute, - double>(BasicType::DOUBLE, CollectionType::SINGLE, "double_sv"); + double>(BasicType::DOUBLE, + CollectionType::SINGLE, + "double_sv"); } TEST_F("Test bitvectors with array value double", BitVectorTest) { f.template test<FloatingPointAttribute, - double>(BasicType::DOUBLE, CollectionType::ARRAY, "double_a"); + double>(BasicType::DOUBLE, + CollectionType::ARRAY, + "double_a"); } TEST_F("Test bitvectors with weighted set value double", BitVectorTest) { f.template test<FloatingPointAttribute, - FloatingPointAttribute::WeightedFloat>(BasicType::DOUBLE, CollectionType::WSET, "double_ws"); + FloatingPointAttribute::WeightedFloat>(BasicType::DOUBLE, + CollectionType::WSET, + "double_ws"); } TEST_F("Test bitvectors with single value string", BitVectorTest) { f.template test<StringAttribute, - vespalib::string>(BasicType::STRING, CollectionType::SINGLE, "string_sv"); + vespalib::string>(BasicType::STRING, + CollectionType::SINGLE, + "string_sv"); } TEST_F("Test bitvectors with array value string", BitVectorTest) { f.template test<StringAttribute, - vespalib::string>(BasicType::STRING, CollectionType::ARRAY, "string_a"); + vespalib::string>(BasicType::STRING, + CollectionType::ARRAY, + "string_a"); } TEST_F("Test bitvectors with weighted set value string", BitVectorTest) { f.template test<StringAttribute, - StringAttribute::WeightedString>(BasicType::STRING, CollectionType::WSET, "string_ws"); + StringAttribute::WeightedString>(BasicType::STRING, + CollectionType::WSET, + "string_ws"); } @@ -569,4 +633,5 @@ TEST("Test that bitvector iterators adheres to SearchIterator requirements") { } } + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index 79ef6e42bb2..c5d70109015 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -123,7 +123,7 @@ void hammerAttribute(AttributePtr &v, DocIdRange range, uint32_t count) Config compactAddressSpaceAttributeConfig(bool enableAddressSpaceCompact) { Config cfg(BasicType::INT8, CollectionType::ARRAY); - cfg.setCompactionStrategy({ 1.0f, (enableAddressSpaceCompact ? 0.2f : 1.0f) }); + cfg.setCompactionStrategy({ 1.0, (enableAddressSpaceCompact ? 0.2 : 1.0) }); return cfg; } diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index 227dbfadbc0..57029f92111 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -4,7 +4,9 @@ #include <vespa/searchcommon/attribute/status.h> #include <vespa/searchlib/attribute/postingstore.h> #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreerootbase.hpp> +#include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/searchlib/attribute/postingstore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/gtest/gtest.h> @@ -40,7 +42,7 @@ std::ostream& operator<<(std::ostream& os, const PostingStoreSetup setup) Config make_config(PostingStoreSetup param) { Config cfg; - cfg.setIsFilter(param.enable_only_bitvector); + cfg.setEnableOnlyBitVector(param.enable_only_bitvector); return cfg; } @@ -210,7 +212,8 @@ PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) EXPECT_EQ(make_exp_sequence(4, 4 + sequence_length), get_sequence(ref1)); EXPECT_EQ(make_exp_sequence(5, 5 + sequence_length), get_sequence(ref2)); auto usage_after = store.getMemoryUsage(); - if ((sequence_length < huge_sequence_length) || !_config.getIsFilter()) { + if (sequence_length < huge_sequence_length || + !_config.getEnableOnlyBitVector()) { EXPECT_GT(usage_before.deadBytes(), usage_after.deadBytes()); } else { EXPECT_EQ(usage_before.deadBytes(), usage_after.deadBytes()); diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index 25de1105973..00e2a82d24e 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -471,7 +471,7 @@ PostingListAttributeTest::checkPostingList(const VectorType & vec, const std::ve } EXPECT_EQ(doc, docEnd); } else { - EXPECT_TRUE(has_bitvector && vec.getIsFilter()); + EXPECT_TRUE(has_bitvector && vec.getEnableOnlyBitVector()); numHits = postingList.getBitVectorEntry(find_result.second)->_bv->reader().countTrueBits(); } if (has_bitvector) { @@ -612,21 +612,21 @@ PostingListAttributeTest::testPostingList(bool enable_only_bitvector, uint32_t n { Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("sint32", cfg); testPostingList<Int32PostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::ARRAY)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("aint32", cfg); testPostingList<Int32ArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::WSET)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("wsint32", cfg); testPostingList<Int32WsetPostingListAttribute>(ptr1, numDocs, values); } @@ -640,21 +640,21 @@ PostingListAttributeTest::testPostingList(bool enable_only_bitvector, uint32_t n { Config cfg(Config(BasicType::FLOAT, CollectionType::SINGLE)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("sfloat", cfg); testPostingList<FloatPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::ARRAY)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("afloat", cfg); testPostingList<FloatArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::WSET)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("wsfloat", cfg); testPostingList<FloatWsetPostingListAttribute>(ptr1, numDocs, values); } @@ -674,21 +674,21 @@ PostingListAttributeTest::testPostingList(bool enable_only_bitvector, uint32_t n { Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("sstr", cfg); testPostingList<StringPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::ARRAY)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("astr", cfg); testPostingList<StringArrayPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::WSET)); cfg.setFastSearch(true); - cfg.setIsFilter(enable_only_bitvector); + cfg.setEnableOnlyBitVector(enable_only_bitvector); AttributePtr ptr1 = create_attribute("wsstr", cfg); testPostingList<StringWsetPostingListAttribute>(ptr1, numDocs, charValues); } diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index e356187a19f..07b64864d9a 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -13,9 +13,9 @@ #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <cinttypes> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP("reference_attribute_test"); @@ -312,8 +312,8 @@ TEST_F(ReferenceAttributeTest, attribute_can_be_saved_and_loaded) assertRef(doc1, 1); assertRef(doc2, 2); assertRef(doc1, 4); - EXPECT_TRUE(vespalib::unlink("test.dat")); - EXPECT_TRUE(vespalib::unlink("test.udat")); + EXPECT_TRUE(std::filesystem::remove(std::filesystem::path("test.dat"))); + EXPECT_TRUE(std::filesystem::remove(std::filesystem::path("test.udat"))); } TEST_F(ReferenceAttributeTest, update_uses_gid_mapper_to_set_target_lid) @@ -399,8 +399,8 @@ TEST_F(ReferenceAttributeTest, populateTargetLids_uses_gid_mapper_to_update_lid_ save(); load(); checkPopulateTargetLids(*this); - EXPECT_TRUE(vespalib::unlink("test.dat")); - EXPECT_TRUE(vespalib::unlink("test.udat")); + EXPECT_TRUE(std::filesystem::remove(std::filesystem::path("test.dat"))); + EXPECT_TRUE(std::filesystem::remove(std::filesystem::path("test.udat"))); } TEST_F(ReferenceAttributeTest, populateTargetLids_handles_removes) diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 9cfd5946dbb..2f3684874ee 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -510,8 +510,8 @@ TEST("require that single weighted set turns filter on filter fields") { SimpleStringTerm node("foo", "", 0, Weight(1)); Result result = do_search(attribute_manager, node, strict); EXPECT_EQUAL(3u, result.est_hits); - EXPECT_TRUE(result.iterator_dump.find("DocumentWeightSearchIterator") != vespalib::string::npos); - EXPECT_TRUE(result.iterator_dump.find("FilterAttributePostingListIteratorT") == vespalib::string::npos); + EXPECT_TRUE(result.iterator_dump.find("DocumentWeightSearchIterator") == vespalib::string::npos); + EXPECT_TRUE(result.iterator_dump.find("FilterAttributePostingListIteratorT") != vespalib::string::npos); ASSERT_EQUAL(3u, result.hits.size()); EXPECT_FALSE(result.est_empty); EXPECT_EQUAL(20u, result.hits[0].docid); diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index ca8eaa176a4..8acb39853e9 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -383,12 +383,6 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire fic.dump(ib); ib.close(); - vespalib::string tsName = dump2dir + "/.teststamp"; - using FileKit = search::FileKit; - ASSERT_TRUE(FileKit::createStamp(tsName)); - ASSERT_TRUE(FileKit::hasStamp(tsName)); - ASSERT_TRUE(FileKit::removeStamp(tsName)); - ASSERT_FALSE(FileKit::hasStamp(tsName)); vespalib::ThreadStackExecutor executor(4); do { diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 5370eff78cf..1d7bd9b8504 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -204,7 +204,7 @@ TEST("test that DirectIOPadding works accordng to spec") { EXPECT_EQUAL(1u, padAfter); EXPECT_TRUE(file.Close()); - FastOS_File::Delete(file.GetFileName()); + std::filesystem::remove(std::filesystem::path(file.GetFileName())); } #endif @@ -748,23 +748,6 @@ TEST("testWriteRead") { std::filesystem::remove_all(std::filesystem::path("empty")); } -TEST("requireThatSyncTokenIsUpdatedAfterFlush") { -#if 0 - std::string file = "sync.dat"; - FastOS_File::Delete(file.c_str()); - { - vespalib::DataBuffer buf; - SimpleDataStore store(file); - EXPECT_EQUAL(0u, store.lastSyncToken()); - makeData(buf, 10); - store.write(0, buf, 10); - store.flush(4); - EXPECT_EQUAL(4u, store.lastSyncToken()); - } - FastOS_File::Delete(file.c_str()); -#endif -} - TEST("requireThatFlushTimeIsAvailableAfterFlush") { DirectoryHandler testDir("flushtime"); vespalib::system_time before(vespalib::system_clock::now()); @@ -1022,7 +1005,7 @@ TEST_F("require that lid space can be increased after being compacted and then s TEST_F("require that there is control of static memory usage", Fixture) { vespalib::MemoryUsage usage = f.store.getMemoryUsage(); - EXPECT_EQUAL(520u + sizeof(LogDataStore::NameIdSet) + sizeof(std::mutex), sizeof(LogDataStore)); + EXPECT_EQUAL(536u + sizeof(LogDataStore::NameIdSet) + sizeof(std::mutex), sizeof(LogDataStore)); EXPECT_EQUAL(74108u, usage.allocatedBytes()); EXPECT_EQUAL(384u, usage.usedBytes()); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index dc64c3328e4..c22d3b3abb8 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -67,6 +67,7 @@ using search::attribute::WeightedEnumContent; using search::attribute::test::AttributeBuilder; using search::common::GeoLocation; using search::common::GeoLocationSpec; +using vespalib::eval::ValueType; using AttributePtr = AttributeVector::SP; using AVC = search::attribute::Config; @@ -391,6 +392,14 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) avs.push_back(AttributeFactory::createAttribute("sbool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 14 avs.push_back(AttributeFactory::createAttribute("sebool", AVC(AVBT::BOOL, AVCT::SINGLE))); // 15 avs.push_back(AttributeFactory::createAttribute("sdouble", AVC(AVBT::DOUBLE, AVCT::SINGLE))); // 16 + { + AVC cfg(AVBT::TENSOR, AVCT::SINGLE); + cfg.setTensorType(ValueType::from_spec("tensor(x[2])")); + avs.push_back(AttributeFactory::createAttribute("tensor", cfg)); + } + avs.push_back(AttributeFactory::createAttribute("predicate", AVC(AVBT::PREDICATE, AVCT::SINGLE))); // 18 + avs.push_back(AttributeFactory::createAttribute("reference", AVC(AVBT::REFERENCE, AVCT::SINGLE))); // 19 + avs.push_back(AttributeFactory::createAttribute("raw", AVC(AVBT::RAW, AVCT::SINGLE))); // 20 // simulate a unique only attribute as specified in sd AVC cfg(AVBT::INT32, AVCT::SINGLE); @@ -417,7 +426,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env) .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL,"sbool") - .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL,"sebool"); + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL,"sebool") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "tensor") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOLEANTREE, "predicate") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::REFERENCE, "reference") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::RAW, "raw"); } for (const auto & attr : avs) { @@ -1499,6 +1512,10 @@ Test::testMatch() ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, "aint"); ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::WEIGHTEDSET, "wsint"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "tensor"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "predicate"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "reference"); + ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "raw"); FtIndexEnvironment idx_env; idx_env.getBuilder() @@ -1507,7 +1524,11 @@ Test::testMatch() .addField(FieldType::INDEX, CollectionType::WEIGHTEDSET, "baz") .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint") .addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, "aint") - .addField(FieldType::ATTRIBUTE, CollectionType::WEIGHTEDSET, "wsint"); + .addField(FieldType::ATTRIBUTE, CollectionType::WEIGHTEDSET, "wsint") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "tensor") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOLEANTREE, "predicate") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::REFERENCE, "reference") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::RAW, "raw"); StringList params, in, out; FT_SETUP_OK(pt, params, in, out.add("score").add("totalWeight")); diff --git a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp index dc9c68c4539..986848c39b7 100644 --- a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp +++ b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp @@ -31,6 +31,7 @@ TEST_F("test default attribute config", Fixture) EXPECT_EQUAL(CollectionType::Type::SINGLE, f._config.collectionType().type()); EXPECT_TRUE(!f._config.fastSearch()); + EXPECT_TRUE(!f._config.getEnableOnlyBitVector()); EXPECT_TRUE(!f._config.getIsFilter()); EXPECT_TRUE(!f._config.fastAccess()); EXPECT_TRUE(f._config.tensorType().is_error()); @@ -42,6 +43,7 @@ TEST_F("test integer weightedset attribute config", EXPECT_EQUAL(BasicType::Type::INT32, f._config.basicType().type()); EXPECT_EQUAL(CollectionType::Type::WSET, f._config.collectionType().type()); EXPECT_TRUE(!f._config.fastSearch()); + EXPECT_TRUE(!f._config.getEnableOnlyBitVector()); EXPECT_TRUE(!f._config.getIsFilter()); EXPECT_TRUE(!f._config.fastAccess()); EXPECT_TRUE(f._config.tensorType().is_error()); diff --git a/searchlib/src/vespa/searchcommon/attribute/basictype.h b/searchlib/src/vespa/searchcommon/attribute/basictype.h index 407348fea92..46387dd2738 100644 --- a/searchlib/src/vespa/searchcommon/attribute/basictype.h +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.h @@ -9,7 +9,7 @@ namespace search::attribute { class BasicType { public: - enum Type : uint8_t { + enum Type { NONE = 0, STRING = 1, BOOL = 2, @@ -28,33 +28,33 @@ class BasicType MAX_TYPE }; - explicit BasicType(int t) noexcept : _type(Type(t)) { } - explicit BasicType(unsigned int t) noexcept : _type(Type(t)) { } - BasicType(Type t) noexcept : _type(t) { } + explicit BasicType(int t) : _type(Type(t)) { } + explicit BasicType(unsigned int t) : _type(Type(t)) { } + BasicType(Type t) : _type(t) { } explicit BasicType(const vespalib::string & t) : _type(asType(t)) { } - Type type() const noexcept { return _type; } - const char * asString() const noexcept { return asString(_type); } - size_t fixedSize() const noexcept { return fixedSize(_type); } - static BasicType fromType(bool) noexcept { return BOOL; } - static BasicType fromType(int8_t) noexcept { return INT8; } - static BasicType fromType(int16_t) noexcept { return INT16; } - static BasicType fromType(int32_t) noexcept { return INT32; } - static BasicType fromType(int64_t) noexcept { return INT64; } - static BasicType fromType(float) noexcept { return FLOAT; } - static BasicType fromType(double) noexcept { return DOUBLE; } - bool operator==(const BasicType &b) const noexcept { return _type == b._type; } - bool operator!=(const BasicType &b) const noexcept { return _type != b._type; } + Type type() const { return _type; } + const char * asString() const { return asString(_type); } + size_t fixedSize() const { return fixedSize(_type); } + static BasicType fromType(bool) { return BOOL; } + static BasicType fromType(int8_t) { return INT8; } + static BasicType fromType(int16_t) { return INT16; } + static BasicType fromType(int32_t) { return INT32; } + static BasicType fromType(int64_t) { return INT64; } + static BasicType fromType(float) { return FLOAT; } + static BasicType fromType(double) { return DOUBLE; } + bool operator==(const BasicType &b) const { return _type == b._type; } + bool operator!=(const BasicType &b) const { return _type != b._type; } private: - static const char * asString(Type t) noexcept { return _typeTable[t]._name; } - static size_t fixedSize(Type t) noexcept { return _typeTable[t]._fixedSize; } + static const char * asString(Type t) { return _typeTable[t]._name; } + static size_t fixedSize(Type t) { return _typeTable[t]._fixedSize; } static Type asType(const vespalib::string & t); Type _type; struct TypeInfo { - Type _type; + Type _type; unsigned int _fixedSize; const char * _name; }; diff --git a/searchlib/src/vespa/searchcommon/attribute/collectiontype.h b/searchlib/src/vespa/searchcommon/attribute/collectiontype.h index 05fad8cbc64..35cb7612ed0 100644 --- a/searchlib/src/vespa/searchcommon/attribute/collectiontype.h +++ b/searchlib/src/vespa/searchcommon/attribute/collectiontype.h @@ -9,7 +9,7 @@ namespace search::attribute { class CollectionType { public: - enum Type : uint8_t { + enum Type { /** * Single value type with one value stored for each document. **/ @@ -26,30 +26,32 @@ class CollectionType MAX_TYPE }; - CollectionType(Type t = SINGLE, bool remove = false, bool create = false) noexcept - : _type(t), - _removeIfZero(remove), - _createIfNonExistant(create) - { } + CollectionType(Type t = SINGLE, bool remove = false, bool create = false) : + _type(t), + _removeIfZero(remove), + _createIfNonExistant(create) + { + } explicit - CollectionType(const vespalib::string & t, bool remove = false, bool create = false) - : _type(asType(t)), - _removeIfZero(remove), - _createIfNonExistant(create) - { } + CollectionType(const vespalib::string & t, bool remove = false, bool create = false) : + _type(asType(t)), + _removeIfZero(remove), + _createIfNonExistant(create) + { + } - Type type() const noexcept { return _type; } - bool isMultiValue() const noexcept { return _type != SINGLE; } - bool isWeightedSet() const noexcept { return _type == WSET; } - bool isArray() const noexcept { return _type == ARRAY; } - bool removeIfZero() const noexcept { return _removeIfZero; } - bool createIfNonExistant() const noexcept { return _createIfNonExistant; } - const char * asString() const noexcept { return asString(_type); } - void removeIfZero(bool newValue) noexcept { _removeIfZero = newValue; } - void createIfNonExistant(bool newValue) noexcept { _createIfNonExistant = newValue; } - bool operator!=(const CollectionType &b) const noexcept { return !(operator==(b)); } - bool operator==(const CollectionType &b) const noexcept { + Type type() const { return _type; } + bool isMultiValue() const { return _type != SINGLE; } + bool isWeightedSet() const { return _type == WSET; } + bool isArray() const { return _type == ARRAY; } + bool removeIfZero() const { return _removeIfZero; } + bool createIfNonExistant() const { return _createIfNonExistant; } + const char * asString() const { return asString(_type); } + void removeIfZero(bool newValue) { _removeIfZero = newValue; } + void createIfNonExistant(bool newValue) { _createIfNonExistant = newValue; } + bool operator!=(const CollectionType &b) const { return !(operator==(b)); } + bool operator==(const CollectionType &b) const { return _type == b._type && _removeIfZero == b._removeIfZero && _createIfNonExistant == b._createIfNonExistant; @@ -61,12 +63,12 @@ class CollectionType const char * _name; }; - static const char * asString(Type t) noexcept { return _typeTable[t]._name; } + static const char * asString(Type t) { return _typeTable[t]._name; } static Type asType(const vespalib::string &t); - Type _type : 4; - bool _removeIfZero : 1; - bool _createIfNonExistant : 1; + Type _type; + bool _removeIfZero; + bool _createIfNonExistant; static const TypeInfo _typeTable[MAX_TYPE]; }; diff --git a/searchlib/src/vespa/searchcommon/attribute/config.cpp b/searchlib/src/vespa/searchcommon/attribute/config.cpp index 7c302a10731..91495025dee 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/config.cpp @@ -19,18 +19,19 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_) noexcept : _basicType(bt), _type(ct), _fastSearch(fastSearch_), + _enableOnlyBitVector(false), _isFilter(false), _fastAccess(false), _mutable(false), _paged(false), - _distance_metric(DistanceMetric::Euclidean), + _maxUnCommittedMemory(MAX_UNCOMMITTED_MEMORY), _match(Match::UNCASED), _dictionary(), - _maxUnCommittedMemory(MAX_UNCOMMITTED_MEMORY), _growStrategy(), _compactionStrategy(), _predicateParams(), _tensorType(vespalib::eval::ValueType::error_type()), + _distance_metric(DistanceMetric::Euclidean), _hnsw_index_params() { } @@ -42,11 +43,12 @@ Config & Config::operator = (Config &&) noexcept = default; Config::~Config() = default; bool -Config::operator==(const Config &b) const noexcept +Config::operator==(const Config &b) const { return _basicType == b._basicType && _type == b._type && _fastSearch == b._fastSearch && + _enableOnlyBitVector == b._enableOnlyBitVector && _isFilter == b._isFilter && _fastAccess == b._fastAccess && _mutable == b._mutable && diff --git a/searchlib/src/vespa/searchcommon/attribute/config.h b/searchlib/src/vespa/searchcommon/attribute/config.h index 17c762267cc..32cac7ec9d6 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.h +++ b/searchlib/src/vespa/searchcommon/attribute/config.h @@ -21,7 +21,7 @@ namespace search::attribute { */ class Config { public: - enum class Match : uint8_t { CASED, UNCASED }; + enum class Match { CASED, UNCASED }; using CompactionStrategy = vespalib::datastore::CompactionStrategy; Config() noexcept; Config(BasicType bt) noexcept : Config(bt, CollectionType::SINGLE) { } @@ -33,27 +33,29 @@ public: Config & operator = (Config &&) noexcept; ~Config(); - BasicType basicType() const noexcept { return _basicType; } - CollectionType collectionType() const noexcept { return _type; } - bool fastSearch() const noexcept { return _fastSearch; } - bool paged() const noexcept { return _paged; } - const PredicateParams &predicateParams() const noexcept { return _predicateParams; } - const vespalib::eval::ValueType & tensorType() const noexcept { return _tensorType; } - DistanceMetric distance_metric() const noexcept { return _distance_metric; } + BasicType basicType() const { return _basicType; } + CollectionType collectionType() const { return _type; } + bool fastSearch() const { return _fastSearch; } + bool paged() const { return _paged; } + const PredicateParams &predicateParams() const { return _predicateParams; } + const vespalib::eval::ValueType & tensorType() const { return _tensorType; } + DistanceMetric distance_metric() const { return _distance_metric; } const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; } /** * Check if attribute posting list can consist of only a bitvector with * no corresponding btree. */ - bool getIsFilter() const noexcept { return _isFilter; } - bool isMutable() const noexcept { return _mutable; } + bool getEnableOnlyBitVector() const { return _enableOnlyBitVector; } + + bool getIsFilter() const { return _isFilter; } + bool isMutable() const { return _mutable; } /** * Check if this attribute should be fast accessible at all times. * If so, attribute is kept in memory also for non-searchable documents. */ - bool fastAccess() const noexcept { return _fastAccess; } + bool fastAccess() const { return _fastAccess; } const GrowStrategy & getGrowStrategy() const { return _growStrategy; } const CompactionStrategy &getCompactionStrategy() const { return _compactionStrategy; } @@ -81,6 +83,14 @@ public: * document frequency goes down, since recreated btree representation * will then have lost weight information. */ + Config & setEnableOnlyBitVector(bool enableOnlyBitVector) { + _enableOnlyBitVector = enableOnlyBitVector; + return *this; + } + + /** + * Hide weight information when searching in attributes. + */ Config & setIsFilter(bool isFilter) { _isFilter = isFilter; return *this; } Config & setMutable(bool isMutable) { _mutable = isMutable; return *this; } Config & setPaged(bool paged_in) { _paged = paged_in; return *this; } @@ -92,28 +102,29 @@ public: } Config & set_dictionary_config(const DictionaryConfig & cfg) { _dictionary = cfg; return *this; } Config & set_match(Match match) { _match = match; return *this; } - bool operator!=(const Config &b) const noexcept { return !(operator==(b)); } - bool operator==(const Config &b) const noexcept ; + bool operator!=(const Config &b) const { return !(operator==(b)); } + bool operator==(const Config &b) const; - uint64_t getMaxUnCommittedMemory() const noexcept { return _maxUnCommittedMemory; } + uint64_t getMaxUnCommittedMemory() const { return _maxUnCommittedMemory; } Config & setMaxUnCommittedMemory(uint64_t value) { _maxUnCommittedMemory = value; return *this; } private: BasicType _basicType; CollectionType _type; - bool _fastSearch : 1; - bool _isFilter : 1; - bool _fastAccess : 1; - bool _mutable : 1; - bool _paged : 1; - DistanceMetric _distance_metric; + bool _fastSearch; + bool _enableOnlyBitVector; + bool _isFilter; + bool _fastAccess; + bool _mutable; + bool _paged; + uint64_t _maxUnCommittedMemory; Match _match; DictionaryConfig _dictionary; - uint64_t _maxUnCommittedMemory; GrowStrategy _growStrategy; CompactionStrategy _compactionStrategy; PredicateParams _predicateParams; vespalib::eval::ValueType _tensorType; + DistanceMetric _distance_metric; std::optional<HnswIndexParams> _hnsw_index_params; }; diff --git a/searchlib/src/vespa/searchcommon/attribute/distance_metric.h b/searchlib/src/vespa/searchcommon/attribute/distance_metric.h index 35f5fb4fe6b..9f9f45810b9 100644 --- a/searchlib/src/vespa/searchcommon/attribute/distance_metric.h +++ b/searchlib/src/vespa/searchcommon/attribute/distance_metric.h @@ -2,10 +2,8 @@ #pragma once -#include <cstdint> - namespace search::attribute { -enum DistanceMetric : uint8_t { Euclidean, Angular, GeoDegrees, InnerProduct, Hamming, PrenormalizedAngular, Dotproduct }; +enum class DistanceMetric { Euclidean, Angular, GeoDegrees, InnerProduct, Hamming, PrenormalizedAngular, Dotproduct }; } diff --git a/searchlib/src/vespa/searchcommon/attribute/persistent_predicate_params.h b/searchlib/src/vespa/searchcommon/attribute/persistent_predicate_params.h index 205a75c188f..d81eb9c5d3c 100644 --- a/searchlib/src/vespa/searchcommon/attribute/persistent_predicate_params.h +++ b/searchlib/src/vespa/searchcommon/attribute/persistent_predicate_params.h @@ -10,23 +10,24 @@ namespace search::attribute { * Persistent parameters for predicate attributes. */ class PersistentPredicateParams { + uint32_t _arity; int64_t _lower_bound; int64_t _upper_bound; - uint32_t _arity; public: - PersistentPredicateParams() noexcept - : _lower_bound(std::numeric_limits<int64_t>::min()), - _upper_bound(std::numeric_limits<int64_t>::max()), - _arity(8) - { } - uint32_t arity() const noexcept { return _arity; } - int64_t lower_bound() const noexcept { return _lower_bound; } - int64_t upper_bound() const noexcept { return _upper_bound; } - void setArity(uint32_t v) noexcept { _arity = v; } - void setBounds(int64_t lower, int64_t upper) noexcept { _lower_bound = lower; _upper_bound = upper; } + PersistentPredicateParams() + : _arity(8), + _lower_bound(std::numeric_limits<int64_t>::min()), + _upper_bound(std::numeric_limits<int64_t>::max()) + { + } + uint32_t arity() const { return _arity; } + int64_t lower_bound() const { return _lower_bound; } + int64_t upper_bound() const { return _upper_bound; } + void setArity(uint32_t v) { _arity = v; } + void setBounds(int64_t lower, int64_t upper) { _lower_bound = lower; _upper_bound = upper; } - bool operator==(const PersistentPredicateParams &rhs) const noexcept { + bool operator==(const PersistentPredicateParams &rhs) const { return ((_arity == rhs._arity) && (_lower_bound == rhs._lower_bound) && (_upper_bound == rhs._upper_bound)); diff --git a/searchlib/src/vespa/searchcommon/attribute/predicate_params.h b/searchlib/src/vespa/searchcommon/attribute/predicate_params.h index 7e9258ab5db..133b7331689 100644 --- a/searchlib/src/vespa/searchcommon/attribute/predicate_params.h +++ b/searchlib/src/vespa/searchcommon/attribute/predicate_params.h @@ -11,16 +11,17 @@ namespace search::attribute { */ class PredicateParams : public PersistentPredicateParams { - float _dense_posting_list_threshold; + double _dense_posting_list_threshold; public: - PredicateParams() noexcept + PredicateParams() : PersistentPredicateParams(), _dense_posting_list_threshold(0.4) - { } + { + } - float dense_posting_list_threshold() const noexcept { return _dense_posting_list_threshold; } - void setDensePostingListThreshold(float v) noexcept { _dense_posting_list_threshold = v; } - bool operator==(const PredicateParams &rhs) const noexcept { + double dense_posting_list_threshold() const { return _dense_posting_list_threshold; } + void setDensePostingListThreshold(double v) { _dense_posting_list_threshold = v; } + bool operator==(const PredicateParams &rhs) const { return (PersistentPredicateParams::operator==(rhs) && (_dense_posting_list_threshold == rhs._dense_posting_list_threshold)); } diff --git a/searchlib/src/vespa/searchcommon/common/dictionary_config.h b/searchlib/src/vespa/searchcommon/common/dictionary_config.h index f504439c5a3..f51341ad799 100644 --- a/searchlib/src/vespa/searchcommon/common/dictionary_config.h +++ b/searchlib/src/vespa/searchcommon/common/dictionary_config.h @@ -3,7 +3,6 @@ #pragma once #include <iosfwd> -#include <cstdint> namespace search { @@ -12,8 +11,8 @@ namespace search { */ class DictionaryConfig { public: - enum class Type : uint8_t { BTREE, HASH, BTREE_AND_HASH }; - enum class Match : uint8_t { CASED, UNCASED }; + enum class Type { BTREE, HASH, BTREE_AND_HASH }; + enum class Match { CASED, UNCASED }; DictionaryConfig() noexcept : _type(Type::BTREE), _match(Match::UNCASED) {} DictionaryConfig(Type type) noexcept : _type(type), _match(Match::UNCASED) {} DictionaryConfig(Type type, Match match) noexcept : _type(type), _match(match) {} @@ -21,8 +20,8 @@ public: Match getMatch() const { return _match; } bool operator == (const DictionaryConfig & b) const { return (_type == b._type) && (_match == b._match); } private: - Type _type : 4; - Match _match : 4; + Type _type; + Match _match; }; std::ostream& operator<<(std::ostream& os, const DictionaryConfig & cfg); diff --git a/searchlib/src/vespa/searchcommon/common/growstrategy.h b/searchlib/src/vespa/searchcommon/common/growstrategy.h index 86750eafbfc..8766989ded0 100644 --- a/searchlib/src/vespa/searchcommon/common/growstrategy.h +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.h @@ -23,17 +23,17 @@ public: { } - static GrowStrategy make(uint32_t docsInitialCapacity, float docsGrowFactor, uint32_t docsGrowDelta) noexcept { + static GrowStrategy make(uint32_t docsInitialCapacity, float docsGrowFactor, uint32_t docsGrowDelta) { return {docsInitialCapacity, docsGrowFactor, docsGrowDelta, 0, 0.2}; } - float getMultiValueAllocGrowFactor() const noexcept { return _multiValueAllocGrowFactor; } + float getMultiValueAllocGrowFactor() const { return _multiValueAllocGrowFactor; } - bool operator==(const GrowStrategy & rhs) const noexcept { + bool operator==(const GrowStrategy & rhs) const { return vespalib::GrowStrategy::operator==(rhs) && (_multiValueAllocGrowFactor == rhs._multiValueAllocGrowFactor); } - bool operator!=(const GrowStrategy & rhs) const noexcept { + bool operator!=(const GrowStrategy & rhs) const { return !(operator==(rhs)); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 399c0266ec9..ba791444dea 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -130,10 +130,26 @@ private: public: AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, - const string &query_stack, const SearchContextParams ¶ms); + const string &query_stack, const SearchContextParams ¶ms) + : AttributeFieldBlueprint(field, attribute, QueryTermDecoder::decodeTerm(query_stack), params) + { } AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, - QueryTermSimple::UP term, const SearchContextParams ¶ms); - ~AttributeFieldBlueprint() override; + QueryTermSimple::UP term, const SearchContextParams ¶ms) + : SimpleLeafBlueprint(field), + _attr(attribute), + _query_term(term->getTermString()), + _search_context(attribute.createSearchContext(std::move(term), params)), + _type(OTHER) + { + uint32_t estHits = _search_context->approximateHits(); + HitEstimate estimate(estHits, estHits == 0); + setEstimate(estimate); + if (attribute.isFloatingPointType()) { + _type = FLOAT; + } else if (attribute.isIntegerType()) { + _type = INT; + } + } SearchIteratorUP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); @@ -165,30 +181,6 @@ public: bool getRange(vespalib::string &from, vespalib::string &to) const override; }; -AttributeFieldBlueprint::~AttributeFieldBlueprint() = default; - -AttributeFieldBlueprint::AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, - const string &query_stack, const SearchContextParams ¶ms) - : AttributeFieldBlueprint(field, attribute, QueryTermDecoder::decodeTerm(query_stack), params) -{ } -AttributeFieldBlueprint::AttributeFieldBlueprint(const FieldSpec &field, const IAttributeVector &attribute, - QueryTermSimple::UP term, const SearchContextParams ¶ms) - : SimpleLeafBlueprint(field), - _attr(attribute), - _query_term(term->getTermString()), - _search_context(attribute.createSearchContext(std::move(term), params)), - _type(OTHER) -{ - uint32_t estHits = _search_context->approximateHits(); - HitEstimate estimate(estHits, estHits == 0); - setEstimate(estimate); - if (attribute.isFloatingPointType()) { - _type = FLOAT; - } else if (attribute.isIntegerType()) { - _type = INT; - } -} - vespalib::string get_type(const IAttributeVector& attr) { @@ -637,11 +629,7 @@ public: return bitvector_iterator; } } - if (_attr.has_weight_iterator(_dict_entry.posting_idx)) { - return std::make_unique<queryeval::DocumentWeightSearchIterator>(*tfmda[0], _attr, _dict_entry); - } else { - return _attr.make_bitvector_iterator(_dict_entry.posting_idx, get_docid_limit(), *tfmda[0], strict); - } + return std::make_unique<queryeval::DocumentWeightSearchIterator>(*tfmda[0], _attr, _dict_entry); } SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index d606daaa3e0..f4ab447ed51 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -125,8 +125,7 @@ bool AttributeVector::hasArrayType() const { return _config->collectionType().is bool AttributeVector::getIsFilter() const { return _config->getIsFilter(); } bool AttributeVector::getIsFastSearch() const { return _config->fastSearch(); } bool AttributeVector::isMutable() const { return _config->isMutable(); } -attribute::BasicType::Type AttributeVector::getBasicType() const { return _config->basicType().type(); } -attribute::CollectionType::Type AttributeVector::getCollectionType() const { return _config->collectionType().type(); } +bool AttributeVector::getEnableOnlyBitVector() const { return _config->getEnableOnlyBitVector(); } bool AttributeVector::isEnumerated(const vespalib::GenericHeader &header) diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 68dfe52643f..e3a7fdeb2c3 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -288,6 +288,7 @@ public: bool getIsFilter() const override final; bool getIsFastSearch() const override final; bool isMutable() const; + bool getEnableOnlyBitVector() const; const Config &getConfig() const noexcept { return *_config; } void update_config(const Config& cfg); @@ -319,8 +320,8 @@ public: AddressSpaceUsage getAddressSpaceUsage() const; - BasicType::Type getBasicType() const override final; - CollectionType::Type getCollectionType() const override final; + BasicType::Type getBasicType() const override final { return getInternalBasicType().type(); } + CollectionType::Type getCollectionType() const override final { return getInternalCollectionType().type(); } uint32_t getCommittedDocIdLimit() const override final { return _committedDocIdLimit.load(std::memory_order_acquire); } bool isImported() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 4e300fe3800..7f04efd940b 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -104,6 +104,7 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) Config retval(bType, cType); PredicateParams predicateParams; retval.setFastSearch(cfg.fastsearch); + retval.setEnableOnlyBitVector(cfg.enableonlybitvector); retval.setIsFilter(cfg.enableonlybitvector); retval.setFastAccess(cfg.fastaccess); retval.setMutable(cfg.ismutable); diff --git a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h index d6499708b76..be36bcd185a 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h @@ -43,7 +43,6 @@ struct IDocumentWeightAttribute virtual void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const = 0; virtual void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const = 0; virtual DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const = 0; - virtual bool has_weight_iterator(vespalib::datastore::EntryRef idx) const noexcept = 0; virtual std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const = 0; virtual ~IDocumentWeightAttribute() = default; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h index f45ba3c8773..71c50ccb270 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h @@ -41,7 +41,6 @@ private: void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const override; DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const override; std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const override; - bool has_weight_iterator(vespalib::datastore::EntryRef idx) const noexcept override; }; DocumentWeightAttributeAdapter _document_weight_attribute_adapter; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index 89ef0a7d8a0..1009fa2fb5f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -146,17 +146,12 @@ MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::make_bi } template <typename B, typename M> -bool -MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::has_weight_iterator(vespalib::datastore::EntryRef idx) const noexcept -{ - return self.getPostingList().has_btree(idx); -} - -template <typename B, typename M> const IDocumentWeightAttribute * MultiValueNumericPostingAttribute<B, M>::asDocumentWeightAttribute() const { - if (this->hasWeightedSetType() && (this->getBasicType() == AttributeVector::BasicType::INT64)) { + if (this->hasWeightedSetType() && + this->getBasicType() == AttributeVector::BasicType::INT64 && + !this->getConfig().getIsFilter()) { return &_document_weight_attribute_adapter; } return nullptr; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index 5c4d97660f6..b25c31a7dea 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -39,7 +39,6 @@ private: void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const override; DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const override; std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const override; - bool has_weight_iterator(vespalib::datastore::EntryRef idx) const noexcept override; }; DocumentWeightAttributeAdapter _document_weight_attribute_adapter; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 3042a9d0bb9..19840b5a474 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -159,13 +159,6 @@ MultiValueStringPostingAttributeT<B, M>::DocumentWeightAttributeAdapter::create( } template <typename B, typename M> -bool -MultiValueStringPostingAttributeT<B, M>::DocumentWeightAttributeAdapter::has_weight_iterator(vespalib::datastore::EntryRef idx) const noexcept -{ - return self.getPostingList().has_btree(idx); -} - -template <typename B, typename M> std::unique_ptr<queryeval::SearchIterator> MultiValueStringPostingAttributeT<B, M>::DocumentWeightAttributeAdapter::make_bitvector_iterator(vespalib::datastore::EntryRef idx, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const { @@ -176,7 +169,9 @@ template <typename B, typename T> const IDocumentWeightAttribute * MultiValueStringPostingAttributeT<B, T>::asDocumentWeightAttribute() const { - if (this->hasWeightedSetType() && (this->getBasicType() == AttributeVector::BasicType::STRING)) { + if (this->hasWeightedSetType() && + this->getBasicType() == AttributeVector::BasicType::STRING && + !this->getConfig().getIsFilter()) { return &_document_weight_attribute_adapter; } return nullptr; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 725491c4702..d32d8cde7ea 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -154,7 +154,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) DocIt postings; vespalib::ConstArrayRef<Posting> array = _merger.getArray(); postings.set(&array[0], &array[array.size()]); - if (_postingList.isFilter()) { + if (_postingList._isFilter) { return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(_baseSearchCtx, matchData, postings); } else { return std::make_unique<AttributePostingListIteratorT<DocIt>>(_baseSearchCtx, _hasWeight, matchData, postings); @@ -182,7 +182,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) DocIt postings; const Posting *array = postingList.getKeyDataEntry(_pidx, clusterSize); postings.set(array, array + clusterSize); - if (postingList.isFilter()) { + if (postingList._isFilter) { return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(_baseSearchCtx, matchData, postings); } else { return std::make_unique<AttributePostingListIteratorT<DocIt>>(_baseSearchCtx, _hasWeight, matchData, postings); @@ -191,7 +191,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) typename PostingList::BTreeType::FrozenView frozen(_frozenRoot, postingList.getAllocator()); using DocIt = typename PostingList::ConstIterator; - if (_postingList.isFilter()) { + if (_postingList._isFilter) { return std::make_unique<FilterAttributePostingListIteratorT<DocIt>>(_baseSearchCtx, matchData, frozen.getRoot(), frozen.getAllocator()); } else { return std::make_unique<AttributePostingListIteratorT<DocIt>> (_baseSearchCtx, _hasWeight, matchData, frozen.getRoot(), frozen.getAllocator()); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 09af15e35d5..2703201b292 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -19,7 +19,9 @@ using vespalib::btree::BTreeNoLeafData; using vespalib::datastore::EntryRefFilter; PostingStoreBase2::PostingStoreBase2(IEnumStoreDictionary& dictionary, Status &status, const Config &config) - : _bvSize(64u), + : _enableOnlyBitVector(config.getEnableOnlyBitVector()), + _isFilter(config.getIsFilter()), + _bvSize(64u), _bvCapacity(128u), _minBvDocFreq(64), _maxBvDocFreq(std::numeric_limits<uint32_t>::max()), @@ -27,9 +29,9 @@ PostingStoreBase2::PostingStoreBase2(IEnumStoreDictionary& dictionary, Status &s _dictionary(dictionary), _status(status), _bvExtraBytes(0), - _compaction_spec(), - _isFilter(config.getIsFilter()) -{ } + _compaction_spec() +{ +} PostingStoreBase2::~PostingStoreBase2() = default; @@ -58,7 +60,8 @@ PostingStoreBase2::resizeBitVectors(uint32_t newSize, uint32_t newCapacity) template <typename DataT> -PostingStore<DataT>::PostingStore(IEnumStoreDictionary& dictionary, Status &status, const Config &config) +PostingStore<DataT>::PostingStore(IEnumStoreDictionary& dictionary, Status &status, + const Config &config) : Parent(false), PostingStoreBase2(dictionary, status, config), _bvType(1, 1024u, RefType::offsetSize()) @@ -182,7 +185,8 @@ PostingStore<DataT>::applyNew(EntryRef &ref, AddIter a, AddIter ae) template <typename DataT> void -PostingStore<DataT>::makeDegradedTree(EntryRef &ref, const BitVector &bv) +PostingStore<DataT>::makeDegradedTree(EntryRef &ref, + const BitVector &bv) { assert(!ref.valid()); BTreeTypeRefPair tPair(allocBTree()); @@ -260,7 +264,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) assert(bv.countTrueBits() == expDocFreq); BitVectorRefPair bPair(allocBitVector()); BitVectorEntry *bve = bPair.data; - if (isFilter()) { + if (_enableOnlyBitVector) { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); _store.hold_entry(ref); @@ -297,7 +301,7 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, AddIter aOrg, AddIter ae) assert(bv.countTrueBits() == expDocFreq); BitVectorRefPair bPair(allocBitVector()); BitVectorEntry *bve = bPair.data; - if (!isFilter()) { + if (!_enableOnlyBitVector) { applyNewTree(bve->_tree, aOrg, ae, CompareT()); } bve->_bv = bvsp; @@ -311,7 +315,11 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, AddIter aOrg, AddIter ae) template <typename DataT> void -PostingStore<DataT>::apply(BitVector &bv, AddIter a, AddIter ae, RemoveIter r, RemoveIter re) +PostingStore<DataT>::apply(BitVector &bv, + AddIter a, + AddIter ae, + RemoveIter r, + RemoveIter re) { while (a != ae || r != re) { if (r != re && (a == ae || *r < a->_key)) { @@ -337,7 +345,11 @@ PostingStore<DataT>::apply(BitVector &bv, AddIter a, AddIter ae, RemoveIter r, R template <typename DataT> void -PostingStore<DataT>::apply(EntryRef &ref, AddIter a, AddIter ae, RemoveIter r, RemoveIter re) +PostingStore<DataT>::apply(EntryRef &ref, + AddIter a, + AddIter ae, + RemoveIter r, + RemoveIter re) { if (!ref.valid()) { // No old data @@ -494,9 +506,11 @@ PostingStore<DataT>::beginFrozen(const EntryRef ref) const return ConstIterator(shortArray, clusterSize, _allocator, _aggrCalc); } + template <typename DataT> void -PostingStore<DataT>::beginFrozen(const EntryRef ref, std::vector<ConstIterator> &where) const +PostingStore<DataT>::beginFrozen(const EntryRef ref, + std::vector<ConstIterator> &where) const { if (!ref.valid()) { where.emplace_back(); @@ -728,7 +742,8 @@ PostingStore<DataT>::compact_worst_buffers(CompactionSpec compaction_spec, const filter.add_buffers(_bvType.get_active_buffers()); } _dictionary.normalize_posting_lists([this](std::vector<EntryRef>& refs) - { return move(refs); }, filter); + { return move(refs); }, + filter); compacting_buffers->finish(); } diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index 8c6ed3d9497..57f523acefe 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -27,7 +27,7 @@ public: std::shared_ptr<GrowableBitVector> _bv; // bitvector public: - BitVectorEntry() noexcept + BitVectorEntry() : _tree(), _bv() { } @@ -36,22 +36,25 @@ public: class PostingStoreBase2 { +public: + bool _enableOnlyBitVector; + bool _isFilter; protected: - static constexpr uint32_t BUFFERTYPE_BITVECTOR = 9u; uint32_t _bvSize; uint32_t _bvCapacity; +public: uint32_t _minBvDocFreq; // Less than this ==> destroy bv uint32_t _maxBvDocFreq; // Greater than or equal to this ==> create bv - std::set<uint32_t> _bvs; // Current bitvectors - IEnumStoreDictionary& _dictionary; - Status &_status; - uint64_t _bvExtraBytes; +protected: + std::set<uint32_t> _bvs; // Current bitvectors + IEnumStoreDictionary& _dictionary; + Status &_status; + uint64_t _bvExtraBytes; PostingStoreCompactionSpec _compaction_spec; -private: - bool _isFilter; + + static constexpr uint32_t BUFFERTYPE_BITVECTOR = 9u; public: - bool isFilter() const noexcept { return _isFilter; } PostingStoreBase2(IEnumStoreDictionary& dictionary, Status &status, const Config &config); virtual ~PostingStoreBase2(); bool resizeBitVectors(uint32_t newSize, uint32_t newCapacity); @@ -108,7 +111,7 @@ public: bool removeSparseBitVectors() override; void consider_remove_sparse_bitvector(std::vector<EntryRef> &refs); - static bool isBitVector(uint32_t typeId) noexcept { return typeId == BUFFERTYPE_BITVECTOR; } + static bool isBitVector(uint32_t typeId) { return typeId == BUFFERTYPE_BITVECTOR; } void applyNew(EntryRef &ref, AddIter a, AddIter ae); @@ -183,9 +186,6 @@ public: BitVectorEntry *getWBitVectorEntry(RefType ref) { return _store.template getEntry<BitVectorEntry>(ref); } - bool has_btree(const EntryRef ref) const noexcept { - return !ref.valid() || !isBitVector(getTypeId(RefType(ref))) || !isFilter(); - } std::unique_ptr<queryeval::SearchIterator> make_bitvector_iterator(RefType ref, uint32_t doc_id_limit, fef::TermFieldMatchData &match_data, bool strict) const; diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp b/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp index 6e56f5477c2..17a0e6256d4 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp +++ b/searchlib/src/vespa/searchlib/attribute/string_search_helper.cpp @@ -41,7 +41,7 @@ StringSearchHelper::StringSearchHelper(StringSearchHelper&&) noexcept = default; StringSearchHelper::~StringSearchHelper() = default; bool -StringSearchHelper::isMatch(const char *src) const noexcept { +StringSearchHelper::isMatch(const char *src) const { if (__builtin_expect(isRegex(), false)) { return getRegex().valid() && getRegex().partial_match(std::string_view(src)); } diff --git a/searchlib/src/vespa/searchlib/attribute/string_search_helper.h b/searchlib/src/vespa/searchlib/attribute/string_search_helper.h index 0c52692ee04..7bfcf0e4292 100644 --- a/searchlib/src/vespa/searchlib/attribute/string_search_helper.h +++ b/searchlib/src/vespa/searchlib/attribute/string_search_helper.h @@ -22,7 +22,7 @@ public: StringSearchHelper(const StringSearchHelper &) = delete; StringSearchHelper & operator =(const StringSearchHelper &) = delete; ~StringSearchHelper(); - bool isMatch(const char *src) const noexcept; + bool isMatch(const char *src) const; bool isPrefix() const noexcept { return _isPrefix; } bool isRegex() const noexcept { return _isRegex; } bool isCased() const noexcept { return _isCased; } diff --git a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp index 60d04d7e3ad..42c9ed4e1c6 100644 --- a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp +++ b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp @@ -4,8 +4,9 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/guard.h> -#include <cassert> #include <algorithm> +#include <cassert> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP(".indexmetainfo"); @@ -305,7 +306,7 @@ IndexMetaInfo::save(const vespalib::string &baseName) { vespalib::string fileName = makeFileName(baseName); vespalib::string newName = fileName + ".new"; - vespalib::unlink(newName); + std::filesystem::remove(std::filesystem::path(newName)); vespalib::FilePointer f(fopen(newName.c_str(), "w")); if (!f.valid()) { LOG(warning, "could not open file for writing: %s", newName.c_str()); diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index 4ae0ce2621a..f4d129bfc58 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -22,7 +22,7 @@ readHeader(vespalib::FileHeader &h, const vespalib::string &name) { Fast_BufferedFile file(32_Ki); - file.OpenReadOnly(name.c_str()); + file.ReadOpenExisting(name.c_str()); h.readFile(file); } @@ -58,8 +58,7 @@ BitVectorFileWrite::open(const vespalib::string &name, if (tuneFileWrite.getWantDirectIO()) { _datFile->EnableDirectIO(); } - // XXX no checking for success: - _datFile->OpenWriteOnly(datname.c_str()); + _datFile->WriteOpen(datname.c_str()); if (_datHeaderLen == 0) { assert(_numKeys == 0); diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index 6913c03262c..0caf89a0730 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -20,7 +20,7 @@ void readHeader(vespalib::FileHeader &h, const vespalib::string &name) { Fast_BufferedFile file(32_Ki); - file.OpenReadOnly(name.c_str()); + file.ReadOpenExisting(name.c_str()); h.readFile(file); } @@ -66,8 +66,7 @@ BitVectorIdxFileWrite::open(const vespalib::string &name, _idxFile->EnableDirectIO(); } - // XXX no checking for success: - _idxFile->OpenWriteOnly(idxname.c_str()); + _idxFile->WriteOpen(idxname.c_str()); if (_idxHeaderLen == 0) { assert(_numKeys == 0); diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp index 24d790afe74..fb1fe98aa88 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp @@ -482,10 +482,6 @@ FieldMerger::merge_field_start() return; } - if (FileKit::hasStamp(_field_dir + "/.mergeocc_done")) { - _state = State::MERGE_DONE; - return; - } std::filesystem::create_directory(std::filesystem::path(_field_dir)); LOG(debug, "merge_field for field %s dir %s", _field_name.c_str(), _field_dir.c_str()); @@ -507,10 +503,6 @@ FieldMerger::merge_field_finish() merge_postings_failed(); return; } - if (!FileKit::createStamp(_field_dir + "/.mergeocc_done")) { - _failed = true; - return; - } vespalib::File::sync(_field_dir); if (!clean_tmp_dirs()) { diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index 7f6f5c4ed15..6d849532931 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -5,8 +5,9 @@ #include "extposocc.h" #include "pagedict4file.h" #include <vespa/vespalib/util/error.h> -#include <vespa/log/log.h> +#include <filesystem> +#include <vespa/log/log.h> LOG_SETUP(".diskindex.fieldwriter"); using search::index::FieldLengthInfo; @@ -184,7 +185,7 @@ FieldWriter::remove(const vespalib::string &prefix) { for (const char **j = termOccNames; *j != nullptr; ++j) { vespalib::string tmpName = prefix + *j; - FastOS_File::Delete(tmpName.c_str()); + std::filesystem::remove(std::filesystem::path(tmpName)); } } diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index 159af42635f..98541dba646 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -15,6 +15,7 @@ #include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/array.hpp> #include <vespa/fastos/file.h> +#include <filesystem> #include <future> #include <vespa/log/log.h> @@ -141,23 +142,14 @@ verifyOrAssert(const TmpChunkMetaV & v) } } -vespalib::string eraseErrorMsg(const vespalib::string & fileName, int error) { - return make_string("Error erasing file '%s'. Error is '%s'", - fileName.c_str(), getErrorString(error).c_str()); -} - } void FileChunk::erase() { _file.reset(); - if (!FastOS_File::Delete(_idxFileName.c_str()) && (errno != ENOENT)) { - throw std::runtime_error(eraseErrorMsg(_idxFileName, errno)); - } - if (!FastOS_File::Delete(_dataFileName.c_str()) && (errno != ENOENT)) { - throw std::runtime_error(eraseErrorMsg(_dataFileName, errno)); - } + std::filesystem::remove(std::filesystem::path(_idxFileName)); + std::filesystem::remove(std::filesystem::path(_dataFileName)); } size_t @@ -569,18 +561,14 @@ void FileChunk::eraseIdxFile(const vespalib::string & name) { vespalib::string fileName(createIdxFileName(name)); - if ( ! FastOS_File::Delete(fileName.c_str())) { - throw std::runtime_error(make_string("Failed to delete '%s'", fileName.c_str())); - } + std::filesystem::remove(std::filesystem::path(fileName)); } void FileChunk::eraseDatFile(const vespalib::string & name) { vespalib::string fileName(createDatFileName(name)); - if ( ! FastOS_File::Delete(fileName.c_str())) { - throw std::runtime_error(make_string("Failed to delete '%s'", fileName.c_str())); - } + std::filesystem::remove(std::filesystem::path(fileName)); } diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp index 53a7ce6e108..7a2148510d4 100644 --- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp @@ -10,9 +10,34 @@ using namespace search::fef; using CollectionType = FieldInfo::CollectionType; +using DataType = FieldInfo::DataType; namespace search::features { +namespace { + +auto attribute_match_data_types = ParameterDataTypeSet::normalTypeSet(); + +bool matchable_field(const FieldInfo& info) +{ + auto field_type = info.type(); + if (field_type != FieldType::INDEX && field_type != FieldType::ATTRIBUTE) { + return false; + } + auto data_type = info.get_data_type(); + if (data_type == DataType::TENSOR || data_type == DataType::RAW) { + // not matchable + return false; + } + if (field_type == FieldType::ATTRIBUTE && !attribute_match_data_types.allowedType(data_type)) { + // bad data type for attributeMatch feature + return false; + } + return true; +} + +} + MatchExecutor::MatchExecutor(const MatchParams & params) : FeatureExecutor(), _params(params) @@ -67,30 +92,28 @@ MatchBlueprint::setup(const IIndexEnvironment & env, { for (uint32_t i = 0; i < env.getNumFields(); ++i) { const FieldInfo * info = env.getField(i); - if (info->get_data_type() == FieldInfo::DataType::TENSOR) { - // not matchable + if (!matchable_field(*info)) { continue; } - if ((info->type() == FieldType::INDEX) || (info->type() == FieldType::ATTRIBUTE)) { - _params.weights.push_back(indexproperties::FieldWeight::lookup(env.getProperties(), info->name())); - if (info->type() == FieldType::INDEX) { - if (info->collection() == CollectionType::SINGLE) { - defineInput("fieldMatch(" + info->name() + ")"); - } else { - defineInput("elementCompleteness(" + info->name() + ")"); - } - } else if (info->type() == FieldType::ATTRIBUTE) { - defineInput("attributeMatch(" + info->name() + ")"); + _params.weights.push_back(indexproperties::FieldWeight::lookup(env.getProperties(), info->name())); + if (info->type() == FieldType::INDEX) { + if (info->collection() == CollectionType::SINGLE) { + defineInput("fieldMatch(" + info->name() + ")"); + } else { + defineInput("elementCompleteness(" + info->name() + ")"); } + } else if (info->type() == FieldType::ATTRIBUTE) { + defineInput("attributeMatch(" + info->name() + ")"); } } describeOutput("score", "Normalized sum over all matched fields"); describeOutput("totalWeight", "Sum of rank weights for all matched fields"); for (uint32_t i = 0; i < env.getNumFields(); ++i) { const FieldInfo * info = env.getField(i); - if ((info->type() == FieldType::INDEX) || (info->type() == FieldType::ATTRIBUTE)) { - describeOutput("weight." + info->name(), "The rank weight value for field '" + info->name() + "'"); + if (!matchable_field(*info)) { + continue; } + describeOutput("weight." + info->name(), "The rank weight value for field '" + info->name() + "'"); } return true; } diff --git a/searchlib/src/vespa/searchlib/fef/properties.cpp b/searchlib/src/vespa/searchlib/fef/properties.cpp index 2cc4e50b593..6f334630dc5 100644 --- a/searchlib/src/vespa/searchlib/fef/properties.cpp +++ b/searchlib/src/vespa/searchlib/fef/properties.cpp @@ -11,7 +11,7 @@ const Property::Value Property::_emptyValue; const Property::Values Property::_emptyValues; const Property::Value & -Property::getAt(uint32_t idx) const noexcept +Property::getAt(uint32_t idx) const { if (idx < (*_values).size()) { return (*_values)[idx]; @@ -22,7 +22,7 @@ Property::getAt(uint32_t idx) const noexcept //----------------------------------------------------------------------------- uint32_t -Properties::rawHash(const void *buf, uint32_t len) noexcept +Properties::rawHash(const void *buf, uint32_t len) { uint32_t res = 0; unsigned const char *pt = (unsigned const char *) buf; @@ -33,7 +33,7 @@ Properties::rawHash(const void *buf, uint32_t len) noexcept return res; } -Properties::Properties() noexcept +Properties::Properties() : _numValues(0), _data() { @@ -59,7 +59,7 @@ Properties::add(vespalib::stringref key, vespalib::stringref value) } uint32_t -Properties::count(vespalib::stringref key) const noexcept +Properties::count(vespalib::stringref key) const { if (!key.empty()) { auto node = _data.find(key); @@ -112,14 +112,14 @@ Properties::clear() } bool -Properties::operator==(const Properties &rhs) const noexcept +Properties::operator==(const Properties &rhs) const { return (_numValues == rhs._numValues && _data == rhs._data); } uint32_t -Properties::hashCode() const noexcept +Properties::hashCode() const { uint32_t hash = numKeys() + numValues(); for (const auto& elem : _data) { @@ -159,7 +159,7 @@ Properties::visitNamespace(vespalib::stringref ns, } Property -Properties::lookup(vespalib::stringref key) const noexcept +Properties::lookup(vespalib::stringref key) const { if (key.empty()) { return Property(); @@ -172,7 +172,7 @@ Properties::lookup(vespalib::stringref key) const noexcept } Property Properties::lookup(vespalib::stringref namespace1, - vespalib::stringref key) const noexcept + vespalib::stringref key) const { if (namespace1.empty() || key.empty()) { return Property(); @@ -184,7 +184,7 @@ Property Properties::lookup(vespalib::stringref namespace1, Property Properties::lookup(vespalib::stringref namespace1, vespalib::stringref namespace2, - vespalib::stringref key) const noexcept + vespalib::stringref key) const { if (namespace1.empty() || namespace2.empty() || key.empty()) { return Property(); @@ -197,7 +197,7 @@ Property Properties::lookup(vespalib::stringref namespace1, Property Properties::lookup(vespalib::stringref namespace1, vespalib::stringref namespace2, vespalib::stringref namespace3, - vespalib::stringref key) const noexcept + vespalib::stringref key) const { if (namespace1.empty() || namespace2.empty() || namespace3.empty() || key.empty()) { return Property(); @@ -207,7 +207,7 @@ Property Properties::lookup(vespalib::stringref namespace1, return lookup(fullKey); } -void Properties::swap(Properties & rhs) noexcept +void Properties::swap(Properties & rhs) { _data.swap(rhs._data); std::swap(_numValues, rhs._numValues); diff --git a/searchlib/src/vespa/searchlib/fef/properties.h b/searchlib/src/vespa/searchlib/fef/properties.h index 80e8c70939c..a6ae83b0339 100644 --- a/searchlib/src/vespa/searchlib/fef/properties.h +++ b/searchlib/src/vespa/searchlib/fef/properties.h @@ -37,7 +37,7 @@ private: * * @param values the values for this property **/ - Property(const Values &values) noexcept : _values(&values) { } + Property(const Values &values) : _values(&values) { } public: /** @@ -46,14 +46,14 @@ public: * object on the stack in the application, and will also be used * by the @ref Properties class when a lookup gives no results. **/ - Property() noexcept : _values(&_emptyValues) { } + Property() : _values(&_emptyValues) { } /** * Check if we found what we were looking for or not. * * @return true if the key we looked up had at least one value **/ - bool found() const noexcept { + bool found() const { return !(*_values).empty(); } @@ -63,7 +63,7 @@ public: * * @return first value for the looked up key, or "" **/ - const Value &get() const noexcept { + const Value &get() const { if ((*_values).empty()) { return _emptyValue; } @@ -78,7 +78,7 @@ public: * @return first value for the looked up key, or fallBack * @param fallBack value to return if no values were found **/ - const Value & get(const Value &fallBack) const noexcept { + const Value & get(const Value &fallBack) const { if ((*_values).empty()) { return fallBack; } @@ -90,7 +90,7 @@ public: * * @return number of values for this property **/ - uint32_t size() const noexcept { return (*_values).size(); } + uint32_t size() const { return (*_values).size(); } /** * Obtain a specific value for the looked up key. @@ -98,7 +98,7 @@ public: * @return the requested value, or "" if idx was out of bounds * @param idx the index of the value we want to access **/ - const Value &getAt(uint32_t idx) const noexcept; + const Value &getAt(uint32_t idx) const; }; //----------------------------------------------------------------------------- @@ -127,7 +127,7 @@ public: /** * Virtual destructor to allow safe subclassing. **/ - virtual ~IPropertiesVisitor() = default; + virtual ~IPropertiesVisitor() {} }; //----------------------------------------------------------------------------- @@ -156,7 +156,7 @@ private: * @param buf data pointer * @param len data length **/ - static uint32_t rawHash(const void *buf, uint32_t len) noexcept; + static uint32_t rawHash(const void *buf, uint32_t len); public: using UP = std::unique_ptr<Properties>; @@ -164,7 +164,7 @@ public: /** * Create an empty properties object. **/ - Properties() noexcept; + Properties(); Properties(Properties &&) noexcept = default; Properties & operator=(Properties &&) noexcept = default; Properties(const Properties &); @@ -192,7 +192,7 @@ public: * @return number of values for the given key * @param key the key **/ - uint32_t count(vespalib::stringref key) const noexcept; + uint32_t count(vespalib::stringref key) const; /** * Remove all values for the given key. @@ -226,14 +226,14 @@ public: * * @return number of keys **/ - uint32_t numKeys() const noexcept { return _data.size(); } + uint32_t numKeys() const { return _data.size(); } /** * Obtain the total number of values stored in this object. * * @return number of values **/ - uint32_t numValues() const noexcept { return _numValues; } + uint32_t numValues() const { return _numValues; } /** * Check if rhs contains the same key/value pairs as this @@ -242,14 +242,14 @@ public: * * @return true if we are equal to rhs **/ - bool operator==(const Properties &rhs) const noexcept; + bool operator==(const Properties &rhs) const; /** * Calculate a hash code for this object * * @return hash code for this object **/ - uint32_t hashCode() const noexcept; + uint32_t hashCode() const; /** * Visit all key/value pairs @@ -275,7 +275,7 @@ public: * @return object encapsulating lookup result * @param key the key to look up **/ - Property lookup(vespalib::stringref key) const noexcept; + Property lookup(vespalib::stringref key) const; /** * Look up a key inside a namespace using the proposed namespace @@ -289,7 +289,7 @@ public: * @param key the key to look up **/ Property lookup(vespalib::stringref namespace1, - vespalib::stringref key) const noexcept; + vespalib::stringref key) const; /** * Look up a key inside a namespace using the proposed namespace @@ -305,7 +305,7 @@ public: **/ Property lookup(vespalib::stringref namespace1, vespalib::stringref namespace2, - vespalib::stringref key) const noexcept; + vespalib::stringref key) const; /** * Look up a key inside a namespace using the proposed namespace @@ -323,13 +323,13 @@ public: Property lookup(vespalib::stringref namespace1, vespalib::stringref namespace2, vespalib::stringref namespace3, - vespalib::stringref key) const noexcept; + vespalib::stringref key) const; - void swap(Properties & rhs) noexcept ; + void swap(Properties & rhs); }; inline void -swap(Properties & a, Properties & b) noexcept +swap(Properties & a, Properties & b) { a.swap(b); } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 3f6085ef7ff..488c58e3119 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -87,7 +87,7 @@ Blueprint::sat_sum(const std::vector<HitEstimate> &data, uint32_t docid_limit) return { uint32_t(std::min(sum, uint64_t(limit))), empty }; } -Blueprint::State::State() noexcept +Blueprint::State::State() : _fields(), _estimateHits(0), _tree_size(1), @@ -97,13 +97,13 @@ Blueprint::State::State() noexcept _cost_tier(COST_TIER_NORMAL) {} -Blueprint::State::State(FieldSpecBase field) noexcept +Blueprint::State::State(FieldSpecBase field) : State() { _fields.add(field); } -Blueprint::State::State(FieldSpecBaseList fields_in) noexcept +Blueprint::State::State(FieldSpecBaseList fields_in) : _fields(std::move(fields_in)), _estimateHits(0), _tree_size(1), @@ -116,7 +116,7 @@ Blueprint::State::State(FieldSpecBaseList fields_in) noexcept Blueprint::State::~State() = default; -Blueprint::Blueprint() noexcept +Blueprint::Blueprint() : _parent(0), _sourceId(0xffffffff), _docid_limit(0), @@ -383,7 +383,7 @@ StateCache::notifyChange() { IntermediateBlueprint::~IntermediateBlueprint() = default; void -IntermediateBlueprint::setDocIdLimit(uint32_t limit) noexcept +IntermediateBlueprint::setDocIdLimit(uint32_t limit) { Blueprint::setDocIdLimit(limit); for (Blueprint::UP &child : _children) { @@ -576,7 +576,7 @@ IntermediateBlueprint::createSearch(fef::MatchData &md, bool strict) const return createIntermediateSearch(std::move(subSearches), strict, md); } -IntermediateBlueprint::IntermediateBlueprint() noexcept = default; +IntermediateBlueprint::IntermediateBlueprint() = default; IntermediateBlueprint & IntermediateBlueprint::addChild(Blueprint::UP child) @@ -737,6 +737,13 @@ LeafBlueprint::optimize(Blueprint* &self) } void +LeafBlueprint::setEstimate(HitEstimate est) +{ + _state.estimate(est); + notifyChange(); +} + +void LeafBlueprint::set_cost_tier(uint32_t value) { assert(value < 0x100); @@ -745,6 +752,13 @@ LeafBlueprint::set_cost_tier(uint32_t value) } void +LeafBlueprint::set_allow_termwise_eval(bool value) +{ + _state.allow_termwise_eval(value); + notifyChange(); +} + +void LeafBlueprint::set_want_global_filter(bool value) { _state.want_global_filter(value); diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 8d230b6ec01..dc7a0992d82 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -48,11 +48,11 @@ public: uint32_t estHits; bool empty; - HitEstimate() noexcept : estHits(0), empty(true) {} - HitEstimate(uint32_t estHits_, bool empty_) noexcept + HitEstimate() : estHits(0), empty(true) {} + HitEstimate(uint32_t estHits_, bool empty_) : estHits(estHits_), empty(empty_) {} - bool operator < (const HitEstimate &other) const noexcept { + bool operator < (const HitEstimate &other) const { if (empty == other.empty) { return (estHits < other.estHits); } else { @@ -77,21 +77,21 @@ public: static constexpr uint8_t COST_TIER_EXPENSIVE = 2; static constexpr uint8_t COST_TIER_MAX = 255; - State() noexcept; - State(FieldSpecBase field) noexcept; - State(FieldSpecBaseList fields_in) noexcept; + State(); + State(FieldSpecBase field); + State(FieldSpecBaseList fields_in); State(const State &rhs) = delete; State(State &&rhs) noexcept = default; State &operator=(const State &rhs) = delete; State &operator=(State &&rhs) noexcept = default; ~State(); - bool isTermLike() const noexcept { return !_fields.empty(); } - const FieldSpecBaseList &fields() const noexcept { return _fields; } + bool isTermLike() const { return !_fields.empty(); } + const FieldSpecBaseList &fields() const { return _fields; } - size_t numFields() const noexcept { return _fields.size(); } - const FieldSpecBase &field(size_t idx) const noexcept { return _fields[idx]; } - const FieldSpecBase *lookupField(uint32_t fieldId) const noexcept { + size_t numFields() const { return _fields.size(); } + const FieldSpecBase &field(size_t idx) const { return _fields[idx]; } + const FieldSpecBase *lookupField(uint32_t fieldId) const { for (const FieldSpecBase & field : _fields) { if (field.getFieldId() == fieldId) { return &field; @@ -100,27 +100,27 @@ public: return nullptr; } - void estimate(HitEstimate est) noexcept { + void estimate(HitEstimate est) { _estimateHits = est.estHits; _estimateEmpty = est.empty; } - HitEstimate estimate() const noexcept { return HitEstimate(_estimateHits, _estimateEmpty); } - double hit_ratio(uint32_t docid_limit) const noexcept { + HitEstimate estimate() const { return HitEstimate(_estimateHits, _estimateEmpty); } + double hit_ratio(uint32_t docid_limit) const { uint32_t total_hits = _estimateHits; uint32_t total_docs = std::max(total_hits, docid_limit); return (total_docs == 0) ? 0.0 : double(total_hits) / double(total_docs); } - void tree_size(uint32_t value) noexcept { + void tree_size(uint32_t value) { assert(value < 0x100000); _tree_size = value; } - uint32_t tree_size() const noexcept { return _tree_size; } - void allow_termwise_eval(bool value) noexcept { _allow_termwise_eval = value; } - bool allow_termwise_eval() const noexcept { return _allow_termwise_eval; } - void want_global_filter(bool value) noexcept { _want_global_filter = value; } - bool want_global_filter() const noexcept { return _want_global_filter; } - void cost_tier(uint8_t value) noexcept { _cost_tier = value; } - uint8_t cost_tier() const noexcept { return _cost_tier; } + uint32_t tree_size() const { return _tree_size; } + void allow_termwise_eval(bool value) { _allow_termwise_eval = value; } + bool allow_termwise_eval() const { return _allow_termwise_eval; } + void want_global_filter(bool value) { _want_global_filter = value; } + bool want_global_filter() const { return _want_global_filter; } + void cost_tier(uint8_t value) { _cost_tier = value; } + uint8_t cost_tier() const { return _cost_tier; } }; // utility that just takes maximum estimate @@ -137,7 +137,7 @@ public: // utility to get the greater estimate to sort first, higher tiers last struct TieredGreaterEstimate { - bool operator () (const auto &a, const auto &b) const noexcept { + bool operator () (const auto &a, const auto &b) const { const auto &lhs = a->getState(); const auto &rhs = b->getState(); if (lhs.cost_tier() != rhs.cost_tier()) { @@ -149,7 +149,7 @@ public: // utility to get the lesser estimate to sort first, higher tiers last struct TieredLessEstimate { - bool operator () (const auto &a, const auto &b) const noexcept { + bool operator () (const auto &a, const auto &b) const { const auto &lhs = a->getState(); const auto &rhs = b->getState(); if (lhs.cost_tier() != rhs.cost_tier()) { @@ -189,20 +189,20 @@ public: // hit that isn't certain to be a match). enum class FilterConstraint { UPPER_BOUND, LOWER_BOUND }; - Blueprint() noexcept; + Blueprint(); Blueprint(const Blueprint &) = delete; Blueprint &operator=(const Blueprint &) = delete; virtual ~Blueprint(); - void setParent(Blueprint *parent) noexcept { _parent = parent; } - Blueprint *getParent() const noexcept { return _parent; } + void setParent(Blueprint *parent) { _parent = parent; } + Blueprint *getParent() const { return _parent; } bool has_parent() const { return (_parent != nullptr); } - Blueprint &setSourceId(uint32_t sourceId) noexcept { _sourceId = sourceId; return *this; } - uint32_t getSourceId() const noexcept { return _sourceId; } + Blueprint &setSourceId(uint32_t sourceId) { _sourceId = sourceId; return *this; } + uint32_t getSourceId() const { return _sourceId; } - virtual void setDocIdLimit(uint32_t limit) noexcept { _docid_limit = limit; } - uint32_t get_docid_limit() const noexcept { return _docid_limit; } + virtual void setDocIdLimit(uint32_t limit) { _docid_limit = limit; } + uint32_t get_docid_limit() const { return _docid_limit; } static Blueprint::UP optimize(Blueprint::UP bp); virtual void optimize(Blueprint* &self) = 0; @@ -227,7 +227,7 @@ public: virtual const State &getState() const = 0; const Blueprint &root() const; - double hit_ratio() const noexcept { return getState().hit_ratio(_docid_limit); } + double hit_ratio() const { return getState().hit_ratio(_docid_limit); } virtual void fetchPostings(const ExecuteInfo &execInfo) = 0; virtual void freeze() = 0; @@ -319,10 +319,10 @@ protected: public: using IndexList = std::vector<size_t>; - IntermediateBlueprint() noexcept; + IntermediateBlueprint(); ~IntermediateBlueprint() override; - void setDocIdLimit(uint32_t limit) noexcept final; + void setDocIdLimit(uint32_t limit) final; void optimize(Blueprint* &self) final; void set_global_filter(const GlobalFilter &global_filter, double estimated_hit_ratio) override; @@ -360,30 +360,24 @@ private: State _state; protected: void optimize(Blueprint* &self) final; - void setEstimate(HitEstimate est) { - _state.estimate(est); - notifyChange(); - } + void setEstimate(HitEstimate est); void set_cost_tier(uint32_t value); - void set_allow_termwise_eval(bool value) { - _state.allow_termwise_eval(value); - notifyChange(); - } + void set_allow_termwise_eval(bool value); void set_want_global_filter(bool value); void set_tree_size(uint32_t value); - LeafBlueprint(bool allow_termwise_eval) noexcept + LeafBlueprint(bool allow_termwise_eval) : _state() { _state.allow_termwise_eval(allow_termwise_eval); } - LeafBlueprint(FieldSpecBase field, bool allow_termwise_eval) noexcept + LeafBlueprint(FieldSpecBase field, bool allow_termwise_eval) : _state(field) { _state.allow_termwise_eval(allow_termwise_eval); } - LeafBlueprint(FieldSpecBaseList fields, bool allow_termwise_eval) noexcept + LeafBlueprint(FieldSpecBaseList fields, bool allow_termwise_eval) : _state(std::move(fields)) { _state.allow_termwise_eval(allow_termwise_eval); @@ -392,7 +386,7 @@ protected: public: ~LeafBlueprint() override = default; const State &getState() const final { return _state; } - void setDocIdLimit(uint32_t limit) noexcept final { Blueprint::setDocIdLimit(limit); } + void setDocIdLimit(uint32_t limit) final { Blueprint::setDocIdLimit(limit); } void fetchPostings(const ExecuteInfo &execInfo) override; void freeze() final; SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const override; @@ -403,15 +397,15 @@ public: // for leaf nodes representing a single term struct SimpleLeafBlueprint : LeafBlueprint { - explicit SimpleLeafBlueprint() noexcept : LeafBlueprint(true) {} - explicit SimpleLeafBlueprint(FieldSpecBase field) noexcept : LeafBlueprint(field, true) {} - explicit SimpleLeafBlueprint(FieldSpecBaseList fields) noexcept: LeafBlueprint(std::move(fields), true) {} + explicit SimpleLeafBlueprint() : LeafBlueprint(true) {} + explicit SimpleLeafBlueprint(FieldSpecBase field) : LeafBlueprint(field, true) {} + explicit SimpleLeafBlueprint(FieldSpecBaseList fields) : LeafBlueprint(std::move(fields), true) {} }; // for leaf nodes representing more complex structures like wand/phrase struct ComplexLeafBlueprint : LeafBlueprint { - explicit ComplexLeafBlueprint(FieldSpecBase field) noexcept : LeafBlueprint(field, false) {} - explicit ComplexLeafBlueprint(FieldSpecBaseList fields) noexcept : LeafBlueprint(std::move(fields), false) {} + explicit ComplexLeafBlueprint(FieldSpecBase field) : LeafBlueprint(field, false) {} + explicit ComplexLeafBlueprint(FieldSpecBaseList fields) : LeafBlueprint(std::move(fields), false) {} }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp b/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp index cd1ddd5b92e..121591723e2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/field_spec.cpp @@ -5,9 +5,9 @@ namespace search::queryeval { -FieldSpecBase::FieldSpecBase(uint32_t fieldId, fef::TermFieldHandle handle, bool isFilter_) noexcept - : _fieldId(fieldId | (isFilter_ ? 0x1000000u : 0)), - _handle(handle) +FieldSpecBase::FieldSpecBase(uint32_t fieldId, fef::TermFieldHandle handle, bool isFilter_) : + _fieldId(fieldId | (isFilter_ ? 0x1000000u : 0)), + _handle(handle) { assert(fieldId < 0x1000000); // Can be represented by 24 bits } diff --git a/searchlib/src/vespa/searchlib/queryeval/field_spec.h b/searchlib/src/vespa/searchlib/queryeval/field_spec.h index c4cd1ac2de8..fd925fdf4ff 100644 --- a/searchlib/src/vespa/searchlib/queryeval/field_spec.h +++ b/searchlib/src/vespa/searchlib/queryeval/field_spec.h @@ -19,15 +19,15 @@ namespace search::queryeval { class FieldSpecBase { public: - FieldSpecBase(uint32_t fieldId, fef::TermFieldHandle handle, bool isFilter_ = false) noexcept; + FieldSpecBase(uint32_t fieldId, fef::TermFieldHandle handle, bool isFilter_ = false); // resolve where to put match information for this term/field combination fef::TermFieldMatchData *resolve(fef::MatchData &md) const; const fef::TermFieldMatchData *resolve(const fef::MatchData &md) const; - uint32_t getFieldId() const noexcept { return _fieldId & 0xffffff; } - fef::TermFieldHandle getHandle() const noexcept { return _handle; } + uint32_t getFieldId() const { return _fieldId & 0xffffff; } + fef::TermFieldHandle getHandle() const { return _handle; } /// a filter produces less detailed match data - bool isFilter() const noexcept { return _fieldId & 0x1000000; } + bool isFilter() const { return _fieldId & 0x1000000; } private: uint32_t _fieldId; // field id in ranking framework fef::TermFieldHandle _handle; // handle used when exposing match data to ranking framework @@ -40,13 +40,13 @@ class FieldSpec : public FieldSpecBase { public: FieldSpec(const vespalib::string & name, uint32_t fieldId, - fef::TermFieldHandle handle, bool isFilter_ = false) noexcept + fef::TermFieldHandle handle, bool isFilter_ = false) : FieldSpecBase(fieldId, handle, isFilter_), _name(name) {} ~FieldSpec(); - const vespalib::string & getName() const noexcept { return _name; } + const vespalib::string & getName() const { return _name; } private: vespalib::string _name; // field name }; @@ -61,7 +61,7 @@ private: List _list; public: - FieldSpecBaseList() noexcept = default; + FieldSpecBaseList() = default; FieldSpecBaseList(FieldSpecBaseList &&) noexcept = default; FieldSpecBaseList & operator=(FieldSpecBaseList &&) noexcept = default; FieldSpecBaseList(const FieldSpecBaseList &) = default; @@ -69,15 +69,15 @@ public: ~FieldSpecBaseList(); void reserve(size_t sz) { _list.reserve(sz); } using const_iterator = const FieldSpecBase *; - FieldSpecBaseList &add(const FieldSpecBase &spec) noexcept { + FieldSpecBaseList &add(const FieldSpecBase &spec) { _list.push_back(spec); return *this; } - bool empty() const noexcept { return _list.empty(); } - size_t size() const noexcept { return _list.size(); } - const_iterator begin() const noexcept { return _list.begin(); } - const_iterator end() const noexcept { return _list.end(); } - const FieldSpecBase &operator[](size_t i) const noexcept { return _list[i]; } + bool empty() const { return _list.empty(); } + size_t size() const { return _list.size(); } + const_iterator begin() const { return _list.begin(); } + const_iterator end() const { return _list.end(); } + const FieldSpecBase &operator[](size_t i) const { return _list[i]; } }; /** @@ -89,7 +89,7 @@ private: vespalib::SmallVector<FieldSpec, 1> _list; public: - FieldSpecList() noexcept = default; + FieldSpecList() = default; FieldSpecList(FieldSpecList &&) noexcept = delete; FieldSpecList & operator=(FieldSpecList &&) noexcept = delete; FieldSpecList(const FieldSpecList &) noexcept = delete; @@ -99,9 +99,9 @@ public: _list.push_back(spec); return *this; } - bool empty() const noexcept { return _list.empty(); } - size_t size() const noexcept { return _list.size(); } - const FieldSpec &operator[](size_t i) const noexcept { return _list[i]; } + bool empty() const { return _list.empty(); } + size_t size() const { return _list.size(); } + const FieldSpec &operator[](size_t i) const { return _list[i]; } void clear() { _list.clear(); } }; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 0cb72202811..aaf361b65ca 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/fastlib/io/bufferedfile.h> #include <cassert> +#include <filesystem> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.domainpart"); @@ -371,7 +372,7 @@ DomainPart::erase(SerialNum to) bool retval(true); if (to > get_range_to()) { close(); - _transLog->Delete(); + std::filesystem::remove(std::filesystem::path(_fileName)); } else { auto range_from = get_range_from(); if (to > range_from) { diff --git a/searchlib/src/vespa/searchlib/util/filekit.cpp b/searchlib/src/vespa/searchlib/util/filekit.cpp index 07eab9bb2be..4012ef00dae 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.cpp +++ b/searchlib/src/vespa/searchlib/util/filekit.cpp @@ -9,87 +9,6 @@ LOG_SETUP(".filekit"); namespace search { -using vespalib::getLastErrorString; - -bool -FileKit::createStamp(const vespalib::string &name) -{ - FastOS_File stamp; - FastOS_StatInfo statInfo; - bool statres; - - statres = FastOS_File::Stat(name.c_str(), &statInfo); - - if (!statres && (statInfo._error != FastOS_StatInfo::FileNotFound)) { - LOG(error, "FATAL: Could not check stamp file %s: %s", - name.c_str(), getLastErrorString().c_str()); - return false; - } - if (statres && statInfo._size > 0) { - LOG(error, "FATAL: Stamp file not empty: %s", name.c_str()); - return false; - } - - if (!stamp.OpenWriteOnlyTruncate(name.c_str())) { - LOG(error, "FATAL: Could not create stamp file %s: %s", - name.c_str(), getLastErrorString().c_str()); - return false; - } - return true; -} - - -bool -FileKit::hasStamp(const vespalib::string &name) -{ - FastOS_StatInfo statInfo; - bool statres; - - statres = FastOS_File::Stat(name.c_str(), &statInfo); - - if (!statres && (statInfo._error != FastOS_StatInfo::FileNotFound)) { - LOG(error, "FATAL: Could not check stamp file %s: %s", - name.c_str(), getLastErrorString().c_str()); - return false; - } - return statres; -} - - -bool -FileKit::removeStamp(const vespalib::string &name) -{ - FastOS_StatInfo statInfo; - bool deleteres; - bool statres; - - statres = FastOS_File::Stat(name.c_str(), &statInfo); - - if (!statres && (statInfo._error != FastOS_StatInfo::FileNotFound)) { - LOG(error, "FATAL: Could not check stamp file %s: %s", - name.c_str(), getLastErrorString().c_str()); - return false; - } - if (statres && statInfo._size > 0) { - LOG(error, "FATAL: Stamp file not empty: %s", name.c_str()); - return false; - } - - do { - deleteres = FastOS_File::Delete(name.c_str()); - //FIX! errno - } while (!deleteres && errno == EINTR); - - if (!deleteres && - FastOS_File::GetLastError() != FastOS_File::ERR_ENOENT) { - LOG(error, "FATAL: Could not remove stamp file %s: %s", - name.c_str(), getLastErrorString().c_str()); - return false; - } - return true; -} - - vespalib::system_time FileKit::getModificationTime(const vespalib::string &name) { diff --git a/searchlib/src/vespa/searchlib/util/filekit.h b/searchlib/src/vespa/searchlib/util/filekit.h index 8c994ff5866..dbd6d2e5a2e 100644 --- a/searchlib/src/vespa/searchlib/util/filekit.h +++ b/searchlib/src/vespa/searchlib/util/filekit.h @@ -10,10 +10,6 @@ namespace search { class FileKit { public: - static bool createStamp(const vespalib::string &name); - static bool hasStamp(const vespalib::string &name); - static bool removeStamp(const vespalib::string &name); - /** * Returns the modification time of the given file/directory, * or time stamp 0 if stating of file/directory fails. |