diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-08-16 13:45:52 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-08-19 07:38:03 +0000 |
commit | d84b3bb9537eac5b0b0e2bb09061ae3d700d7199 (patch) | |
tree | 97fd1611e8adb6ffbee3426a147438482e7e38ba | |
parent | c3ece362865b788a6fb9a86718cb34ec9af609aa (diff) |
Remove "enable enumerated save" flag from attribute vectors as this should be always on.
For production code this has been on for several years already.
10 files changed, 106 insertions, 182 deletions
diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h index 6dbf311ef82..71319d782f1 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h @@ -5,6 +5,7 @@ #include "collectiontype.h" #include "basictype.h" #include <vespa/searchcommon/common/iblobconverter.h> +#include <ostream> #include <vector> namespace search { @@ -48,6 +49,14 @@ public: } }; +template <typename T> +std::ostream& +operator<<(std::ostream& os, const WeightedType<T>& value) +{ + os << "{" << value.value() << "," << value.weight() << "}"; + return os; +} + /** * This is a read interface used to access the content of an attribute vector. **/ diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp index 7d91538f657..fb74d8f27f9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_factory.cpp @@ -15,7 +15,6 @@ AttributeVector::SP AttributeFactory::create(const vespalib::string &name, const search::attribute::Config &cfg) const { AttributeVector::SP v(search::AttributeFactory::createAttribute(name, cfg)); - v->enableEnumeratedSave(true); return v; } diff --git a/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp index 3f9a784d57a..1834beb9590 100644 --- a/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp +++ b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp @@ -111,14 +111,13 @@ LoadAttribute::Main() bool doApplyUpdate = false; bool doSave = false; bool doFastSearch = false; - bool doEnableEnumeratedSave = false; bool doHuge = false; int idx = 1; char opt; const char * arg; bool optError = false; - while ((opt = GetOpt("pasf:eh", arg, idx)) != -1) { + while ((opt = GetOpt("pasf:h", arg, idx)) != -1) { switch (opt) { case 'p': doPrintContent = true; @@ -126,9 +125,6 @@ LoadAttribute::Main() case 'a': doApplyUpdate = true; break; - case 'e': - doEnableEnumeratedSave = true; - break; case 'h': doHuge = true; break; @@ -169,9 +165,6 @@ LoadAttribute::Main() c.setFastSearch(doFastSearch); c.setHuge(doHuge); AttributePtr ptr = AttributeFactory::createAttribute(fileName, c); - if (doEnableEnumeratedSave) { - ptr->enableEnumeratedSave(); - } FastOS_Time timer; timer.SetNow(); load(ptr); diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index c71e51b2040..8259d5ba0f8 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -107,9 +107,8 @@ statSize(const AttributeVector &a) bool preciseEstimatedSize(const AttributeVector &a) { - if (a.getBasicType() == BasicType::STRING && - EXPECT_TRUE(a.hasEnum()) && !a.getEnumeratedSave()) { - return false; // Using average of string lens, can be somewhat off + if (a.getBasicType() == BasicType::STRING) { + return false; // Using average of string lengths, can be somewhat off } return true; } @@ -217,7 +216,7 @@ private: template <typename VectorType, typename BufferType> void - testCompactLidSpace(const Config &config, bool fs, bool es); + testCompactLidSpace(const Config &config, bool fast_search); template <typename VectorType, typename BufferType> void @@ -399,7 +398,7 @@ void AttributeTest::compare(VectorType & a, VectorType & b) EXPECT_EQUAL(static_cast<const AttributeVector &>(a).get(i, av, asz), static_cast<uint32_t>(a.getValueCount(i))); EXPECT_EQUAL(static_cast<const AttributeVector &>(b).get(i, bv, bsz), static_cast<uint32_t>(b.getValueCount(i))); for(size_t j(0), k(std::min(a.getValueCount(i), b.getValueCount(i))); j < k; j++) { - EXPECT_TRUE(av[j] == bv[j]); + EXPECT_EQUAL(av[j], bv[j]); } } delete [] bv; @@ -463,48 +462,13 @@ void AttributeTest::testReload(const AttributePtr & a, const AttributePtr & b, c compare<VectorType, BufferType> (*(static_cast<VectorType *>(a.get())), *(static_cast<VectorType *>(c.get()))); - if (isUnsignedSmallIntAttribute(*a)) + if (isUnsignedSmallIntAttribute(*a)) { return; + } populate(static_cast<VectorType &>(*b.get()), 700); populate(static_cast<VectorType &>(*c.get()), 700); compare<VectorType, BufferType> (*(static_cast<VectorType *>(b.get())), *(static_cast<VectorType *>(c.get()))); - - { - ReadAttributeFile readC(c->getBaseFileName(), c->getConfig()); - WriteAttributeFile writeC(b->getBaseFileName(), b->getConfig(), - DummyFileHeaderContext(), - c->getNumDocs()); - std::unique_ptr<AttributeFile::Record> record(readC.getRecord()); - ASSERT_TRUE(record.get()); - for (size_t i(0), m(c->getNumDocs()); i < m; i++) { - EXPECT_TRUE(readC.read(*record)); - EXPECT_TRUE(writeC.write(*record)); - } - EXPECT_TRUE( ! readC.read(*record)); - } - EXPECT_TRUE( b->load() ); - compare<VectorType, BufferType> - (*(static_cast<VectorType *>(a.get())), - *(static_cast<VectorType *>(b.get()))); - { - ReadAttributeFile readC(c->getBaseFileName(), c->getConfig()); - WriteAttributeFile writeC(b->getBaseFileName(), b->getConfig(), - DummyFileHeaderContext(), - c->getNumDocs()); - readC.enableDirectIO(); - writeC.enableDirectIO(); - std::unique_ptr<AttributeFile::Record> record(readC.getRecord()); - ASSERT_TRUE(record.get()); - for (size_t i(0), m(c->getNumDocs()); i < m; i++) { - EXPECT_TRUE(readC.read(*record)); - EXPECT_TRUE(writeC.write(*record)); - } - EXPECT_TRUE( ! readC.read(*record)); - } - EXPECT_TRUE( b->load() ); - compare<VectorType, BufferType> - (*(static_cast<VectorType *>(a.get())), *(static_cast<VectorType *>(b.get()))); } @@ -1949,28 +1913,24 @@ AttributeTest::testPredicateHeaderTags() template <typename VectorType, typename BufferType> void AttributeTest::testCompactLidSpace(const Config &config, - bool fs, - bool es) + bool fast_search) { uint32_t highDocs = 100; uint32_t trimmedDocs = 30; vespalib::string bts = config.basicType().asString(); vespalib::string cts = config.collectionType().asString(); - vespalib::string fas = fs ? "-fs" : ""; - vespalib::string ess = es ? "-es" : ""; + vespalib::string fas = fast_search ? "-fs" : ""; Config cfg = config; - cfg.setFastSearch(fs); + cfg.setFastSearch(fast_search); - vespalib::string name = clsDir + "/" + bts + "-" + cts + fas + ess; + vespalib::string name = clsDir + "/" + bts + "-" + cts + fas; LOG(info, "testCompactLidSpace(%s)", name.c_str()); AttributePtr attr = AttributeFactory::createAttribute(name, cfg); auto &v = static_cast<VectorType &>(*attr.get()); - attr->enableEnumeratedSave(es); attr->addDocs(highDocs); populate(v, 17); AttributePtr attr2 = AttributeFactory::createAttribute(name, cfg); auto &v2 = static_cast<VectorType &>(*attr2.get()); - attr2->enableEnumeratedSave(es); attr2->addDocs(trimmedDocs); populate(v2, 17); EXPECT_EQUAL(trimmedDocs, attr2->getNumDocs()); @@ -1999,13 +1959,12 @@ template <typename VectorType, typename BufferType> void AttributeTest::testCompactLidSpace(const Config &config) { - testCompactLidSpace<VectorType, BufferType>(config, false, false); - testCompactLidSpace<VectorType, BufferType>(config, false, true); + testCompactLidSpace<VectorType, BufferType>(config, false); bool smallUInt = isUnsignedSmallIntAttribute(config.basicType().type()); - if (smallUInt) + if (smallUInt) { return; - testCompactLidSpace<VectorType, BufferType>(config, true, false); - testCompactLidSpace<VectorType, BufferType>(config, true, true); + } + testCompactLidSpace<VectorType, BufferType>(config, true); } void diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index d4dc3adfcea..de6c8072b61 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -150,7 +150,7 @@ private: SearchContextPtr getSearch(const V & vec); MemAttr::SP saveMem(AttributeVector &v); - void checkMem(AttributeVector &v, const MemAttr &e, bool enumerated); + void checkMem(AttributeVector &v, const MemAttr &e); MemAttr::SP saveBoth(AttributePtr v); AttributePtr make(Config cfg, const vespalib::string &pref, bool fastSearch = false); void load(AttributePtr v, const vespalib::string &name); @@ -513,13 +513,10 @@ EnumeratedSaveTest::saveMem(AttributeVector &v) void -EnumeratedSaveTest::checkMem(AttributeVector &v, const MemAttr &e, - bool enumerated) +EnumeratedSaveTest::checkMem(AttributeVector &v, const MemAttr &e) { MemAttr m; - v.enableEnumeratedSave(enumerated); EXPECT_TRUE(v.save(m, v.getBaseFileName())); - v.enableEnumeratedSave(false); ASSERT_TRUE(m == e); } @@ -531,16 +528,14 @@ EnumeratedSaveTest::saveBoth(AttributePtr v) vespalib::string basename = v->getBaseFileName(); AttributePtr v2 = make(v->getConfig(), basename, true); EXPECT_TRUE(v2->load()); - v2->enableEnumeratedSave(true); EXPECT_TRUE(v2->save(basename + "_e")); - if ((v->getConfig().basicType() == BasicType::INT32 && - v->getConfig().collectionType() == CollectionType::WSET) || true) { - search::AttributeMemorySaveTarget ms; - search::TuneFileAttributes tune; - search::index::DummyFileHeaderContext fileHeaderContext; - EXPECT_TRUE(v2->save(ms, basename + "_ee")); - EXPECT_TRUE(ms.writeToFile(tune, fileHeaderContext)); - } + + search::AttributeMemorySaveTarget ms; + search::TuneFileAttributes tune; + search::index::DummyFileHeaderContext fileHeaderContext; + EXPECT_TRUE(v2->save(ms, basename + "_ee")); + EXPECT_TRUE(ms.writeToFile(tune, fileHeaderContext)); + return saveMem(*v2); } @@ -602,34 +597,24 @@ EnumeratedSaveTest::testReload(AttributePtr v0, TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "0", v0))); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "1", v1))); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "2", v2))); - TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "1", v1))); - TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "0", v0))); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "0", v0))); - TEST_DO(checkMem(*v, *mv0, false)); - TEST_DO(checkMem(*v, supportsEnumerated ? *emv0 : *mv0, true)); + TEST_DO(checkMem(*v, supportsEnumerated ? *emv0 : *mv0)); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "1", v1))); - TEST_DO(checkMem(*v, *mv1, false)); - TEST_DO(checkMem(*v, supportsEnumerated ? *emv1 : *mv1, true)); + TEST_DO(checkMem(*v, supportsEnumerated ? *emv1 : *mv1)); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "2", v2))); - TEST_DO(checkMem(*v, *mv2, false)); - TEST_DO(checkMem(*v, supportsEnumerated ? *emv2 : *mv2, true)); + TEST_DO(checkMem(*v, supportsEnumerated ? *emv2 : *mv2)); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "0_e", v0))); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "1_e", v1))); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "2_e", v2))); - TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "1_e", v1))); - TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "0_e", v0))); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "0_e", v0))); - TEST_DO(checkMem(*v, *mv0, false)); - TEST_DO(checkMem(*v, supportsEnumerated ? *emv0 : *mv0, true)); + TEST_DO(checkMem(*v, supportsEnumerated ? *emv0 : *mv0)); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "1_e", v1))); - TEST_DO(checkMem(*v, *mv1, false)); - TEST_DO(checkMem(*v, supportsEnumerated ? *emv1 : *mv1, true)); + TEST_DO(checkMem(*v, supportsEnumerated ? *emv1 : *mv1)); TEST_DO((checkLoad<VectorType, BufferType>(v, pref + "2_e", v2))); - TEST_DO(checkMem(*v, *mv2, false)); - TEST_DO(checkMem(*v, supportsEnumerated ? *emv2 : *mv2, true)); + TEST_DO(checkMem(*v, supportsEnumerated ? *emv2 : *mv2)); TermFieldMatchData md; SearchContextPtr sc = getSearch<VectorType>(as<VectorType>(v)); diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index c6270e32c85..bc72caf198b 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -75,12 +75,12 @@ assertIterator(const std::string &exp, SearchIterator &it, return true; } +using AttributePtr = AttributeVector::SP; class PostingListAttributeTest : public vespalib::TestApp { private: typedef IntegerAttribute::largeint_t largeint_t; - typedef AttributeVector::SP AttributePtr; typedef std::set<AttributeVector::DocId> DocSet; typedef SingleValueNumericPostingAttribute< @@ -142,8 +142,7 @@ private: void checkSearch(bool useBitVector, const AttributeVector & vec, const BufferType & term, uint32_t numHits, uint32_t docBegin, uint32_t docEnd); template <typename VectorType, typename BufferType> - void testPostingList(const AttributePtr & ptr1, const AttributePtr & ptr2, - uint32_t numDocs, const std::vector<BufferType> & values); + void testPostingList(const AttributePtr& ptr1, uint32_t numDocs, const std::vector<BufferType>& values); void testPostingList(); void testPostingList(bool enableBitVector); void testPostingList(bool enableBitVector, uint32_t numDocs, uint32_t numUniqueValues); @@ -506,15 +505,20 @@ PostingListAttributeTest::checkSearch(bool useBitVector, const AttributeVector & EXPECT_EQUAL(docEnd, lastDocId+1); } +AttributePtr +create_as(const AttributeVector& attr, const std::string& name_suffix) +{ + return AttributeFactory::createAttribute(attr.getName() + name_suffix, attr.getConfig()); +} + template <typename VectorType, typename BufferType> void -PostingListAttributeTest::testPostingList(const AttributePtr & ptr1, const AttributePtr & ptr2, - uint32_t numDocs, const std::vector<BufferType> & values) +PostingListAttributeTest::testPostingList(const AttributePtr& ptr1, uint32_t numDocs, + const std::vector<BufferType>& values) { LOG(info, "testPostingList: vector '%s'", ptr1->getName().c_str()); - VectorType & vec1 = static_cast<VectorType &>(*ptr1.get()); - VectorType & vec2 = static_cast<VectorType &>(*ptr2.get()); + auto& vec1 = static_cast<VectorType &>(*ptr1.get()); addDocs(ptr1, numDocs); uint32_t part = numDocs / values.size(); @@ -526,28 +530,14 @@ PostingListAttributeTest::testPostingList(const AttributePtr & ptr1, const Attri } vec1.commit(); -#if 0 - std::cout << "***** printBuffer 0 ***** " << std::endl; - vec1.getEnumStore().printBuffer(std::cout, 0); - std::cout << "***** printBuffer 1 ***** " << std::endl; - vec1.getEnumStore().printBuffer(std::cout, 1); - std::cout << "***** printCurrentContent ***** " << std::endl; - vec1.getEnumStore().printCurrentContent(std::cout); - std::cout << "***** printPostingListContent *****" << std::endl; - vec1.printPostingListContent(std::cout); -#endif - // check posting list for correct content checkPostingList(vec1, values, RangeAlpha(part)); // load and save vector + auto ptr2 = create_as(*ptr1, "_2"); ptr1->save(ptr2->getBaseFileName()); ptr2->load(); -#if 0 - std::cout << "***** vec2.printPostingListContent *****" << std::endl; - vec2.printPostingListContent(std::cout); -#endif - checkPostingList(vec2, values, RangeAlpha(part)); + checkPostingList(static_cast<VectorType&>(*ptr2.get()), values, RangeAlpha(part)); // insert values in another order for (uint32_t doc = 0; doc < numDocs; ++doc) { @@ -560,9 +550,10 @@ PostingListAttributeTest::testPostingList(const AttributePtr & ptr1, const Attri checkPostingList(vec1, values, RangeBeta(part, values.size())); // load and save vector - ptr1->save(ptr2->getBaseFileName()); - ptr2->load(); - checkPostingList(vec2, values, RangeBeta(part, values.size())); + auto ptr3 = create_as(*ptr1, "_3"); + ptr1->save(ptr3->getBaseFileName()); + ptr3->load(); + checkPostingList(static_cast<VectorType&>(*ptr3.get()), values, RangeBeta(part, values.size())); } void @@ -592,25 +583,22 @@ PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("sint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sint32_2", cfg); - testPostingList<Int32PostingListAttribute>(ptr1, ptr2, numDocs, values); + AttributePtr ptr1 = AttributeFactory::createAttribute("sint32", cfg); + testPostingList<Int32PostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::ARRAY)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("aint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("aint32_2", cfg); - testPostingList<Int32ArrayPostingListAttribute>(ptr1, ptr2, numDocs, values); + AttributePtr ptr1 = AttributeFactory::createAttribute("aint32", cfg); + testPostingList<Int32ArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::WSET)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("wsint32_2", cfg); - testPostingList<Int32WsetPostingListAttribute>(ptr1, ptr2, numDocs, values); + AttributePtr ptr1 = AttributeFactory::createAttribute("wsint32", cfg); + testPostingList<Int32WsetPostingListAttribute>(ptr1, numDocs, values); } } @@ -623,25 +611,22 @@ PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs Config cfg(Config(BasicType::FLOAT, CollectionType::SINGLE)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("sfloat_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sfloat_2", cfg); - testPostingList<FloatPostingListAttribute>(ptr1, ptr2, numDocs, values); + AttributePtr ptr1 = AttributeFactory::createAttribute("sfloat", cfg); + testPostingList<FloatPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::ARRAY)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("afloat_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("afloat_2", cfg); - testPostingList<FloatArrayPostingListAttribute>(ptr1, ptr2, numDocs, values); + AttributePtr ptr1 = AttributeFactory::createAttribute("afloat", cfg); + testPostingList<FloatArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::WSET)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsfloat_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("wsfloat_2", cfg); - testPostingList<FloatWsetPostingListAttribute>(ptr1, ptr2, numDocs, values); + AttributePtr ptr1 = AttributeFactory::createAttribute("wsfloat", cfg); + testPostingList<FloatWsetPostingListAttribute>(ptr1, numDocs, values); } } @@ -660,25 +645,22 @@ PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("sstr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sstr_2", cfg); - testPostingList<StringPostingListAttribute>(ptr1, ptr2, numDocs, charValues); + AttributePtr ptr1 = AttributeFactory::createAttribute("sstr", cfg); + testPostingList<StringPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::ARRAY)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("astr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("astr_2", cfg); - testPostingList<StringArrayPostingListAttribute>(ptr1, ptr2, numDocs, charValues); + AttributePtr ptr1 = AttributeFactory::createAttribute("astr", cfg); + testPostingList<StringArrayPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::WSET)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsstr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("wsstr_2", cfg); - testPostingList<StringWsetPostingListAttribute>(ptr1, ptr2, numDocs, charValues); + AttributePtr ptr1 = AttributeFactory::createAttribute("wsstr", cfg); + testPostingList<StringWsetPostingListAttribute>(ptr1, numDocs, charValues); } } } diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp index 574b8f34caf..8329398f8af 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp @@ -158,7 +158,7 @@ private: template <typename V, typename T> void testMultiValueSearchHelper(V & vec, const std::vector<T> & values); template <typename V, typename T> - void testMultiValueSearch(V & first, V & second, const std::vector<T> & values); + void testMultiValueSearch(V& attr, uint32_t num_docs, const std::vector<T> & values); void testSearch(); class IteratorTester { @@ -555,22 +555,30 @@ SearchContextTest::testMultiValueSearchHelper(V & vec, const std::vector<T> & va } } +AttributePtr +create_as(const AttributeVector& attr, const std::string& name_suffix) +{ + return AttributeFactory::createAttribute(attr.getName() + name_suffix, attr.getConfig()); +} + + template <typename V, typename T> void -SearchContextTest::testMultiValueSearch(V & first, V & second, const std::vector<T> & values) +SearchContextTest::testMultiValueSearch(V& attr, uint32_t num_docs, const std::vector<T> & values) { - addDocs(first, second.getNumDocs()); + addDocs(attr, num_docs); LOG(info, "testMultiValueSearch: vector '%s' with %u documents and %lu unique values", - first.getName().c_str(), first.getNumDocs(), values.size()); + attr.getName().c_str(), attr.getNumDocs(), values.size()); - fillAttribute(first, values); + fillAttribute(attr, values); - testMultiValueSearchHelper(first, values); + testMultiValueSearchHelper(attr, values); - ASSERT_TRUE(first.save(second.getBaseFileName())); - ASSERT_TRUE(second.load()); + auto attr2 = create_as(attr, "_2"); + ASSERT_TRUE(attr.save(attr2->getBaseFileName())); + ASSERT_TRUE(attr2->load()); - testMultiValueSearchHelper(second, values); + testMultiValueSearchHelper(static_cast<V&>(*attr2.get()), values); size_t sz = values.size(); ASSERT_TRUE(sz > 2); @@ -581,18 +589,19 @@ SearchContextTest::testMultiValueSearch(V & first, V & second, const std::vector subset.push_back(values[i]); } - fillAttribute(first, subset); + fillAttribute(attr, subset); - ASSERT_TRUE(1u < first.getNumDocs()); - EXPECT_TRUE(first.append(1u, values[sz - 1], 1)); - first.commit(true); + ASSERT_TRUE(1u < attr.getNumDocs()); + EXPECT_TRUE(attr.append(1u, values[sz - 1], 1)); + attr.commit(true); - testMultiValueSearchHelper(first, values); + testMultiValueSearchHelper(attr, values); - ASSERT_TRUE(first.save(second.getBaseFileName())); - ASSERT_TRUE(second.load()); + auto attr3 = create_as(attr, "_3"); + ASSERT_TRUE(attr.save(attr3->getBaseFileName())); + ASSERT_TRUE(attr3->load()); - testMultiValueSearchHelper(second, values); + testMultiValueSearchHelper(static_cast<V&>(*attr3.get()), values); } template<typename T, typename A> @@ -606,8 +615,7 @@ void SearchContextTest::testSearch(const ConfigMap & cfgs) { testSearch(*(dynamic_cast<A *>(second.get())), numDocs, values); if (second->hasMultiValue()) { AttributePtr first = AttributeFactory::createAttribute(cfg.first + "-1", cfg.second); - testMultiValueSearch(*(dynamic_cast<A *>(first.get())), - *(dynamic_cast<A *>(second.get())), values); + testMultiValueSearch(*(dynamic_cast<A *>(first.get())), second->getNumDocs(), values); } } } @@ -707,8 +715,7 @@ SearchContextTest::testSearch() AttributePtr second = AttributeFactory::createAttribute("flags-2", cfg); testSearch(*(dynamic_cast<IntegerAttribute *>(second.get())), numDocs, values); AttributePtr first = AttributeFactory::createAttribute("flags-1", cfg); - testMultiValueSearch(*(dynamic_cast<IntegerAttribute *>(first.get())), - *(dynamic_cast<IntegerAttribute *>(second.get())), values); + testMultiValueSearch(*(dynamic_cast<IntegerAttribute *>(first.get())), second->getNumDocs(), values); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 130354efa60..3cefb665181 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -127,9 +127,9 @@ AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config _createSerialNum(0u), _compactLidSpaceGeneration(0u), _hasEnum(false), - _loaded(false), - _enableEnumeratedSave(false) -{ } + _loaded(false) +{ +} AttributeVector::~AttributeVector() = default; @@ -595,13 +595,6 @@ AttributeVector::addReservedDoc() } } - -void -AttributeVector::enableEnumeratedSave(bool enable) { - if (hasEnum() || !enable) - _enableEnumeratedSave = enable; -} - attribute::IPostingListAttributeBase *AttributeVector::getIPostingListAttributeBase() { return nullptr; } const attribute::IPostingListAttributeBase *AttributeVector::getIPostingListAttributeBase() const { return nullptr; } const IDocumentWeightAttribute * AttributeVector::asDocumentWeightAttribute() const { return nullptr; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 8d902dd9d74..20803ca2387 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -583,7 +583,6 @@ private: uint64_t _compactLidSpaceGeneration; bool _hasEnum; bool _loaded; - bool _enableEnumeratedSave; fastos::TimeStamp _nextStatUpdateTime; ////// Locking strategy interface. only available from the Guards. @@ -631,8 +630,7 @@ public: * Add reserved initial document with docId 0 and undefined value. */ void addReservedDoc(); - void enableEnumeratedSave(bool enable = true); - bool getEnumeratedSave() const { return _hasEnum && _enableEnumeratedSave; } + bool getEnumeratedSave() const { return _hasEnum; } virtual attribute::IPostingListAttributeBase * getIPostingListAttributeBase(); virtual const attribute::IPostingListAttributeBase * getIPostingListAttributeBase() const; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 05ea57cf43a..8b1b272be52 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -48,7 +48,6 @@ ReferenceAttribute::ReferenceAttribute(const vespalib::stringref baseFileName, _referenceMappings(getGenerationHolder(), getCommittedDocIdLimitRef()) { setEnum(true); - enableEnumeratedSave(true); } ReferenceAttribute::~ReferenceAttribute() |