diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-08-28 11:27:29 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-08-28 11:27:29 +0000 |
commit | 54db8581b0b8c6979681d8bd06afd1fc87cf8357 (patch) | |
tree | 2076451c5fcb9943e9d6d715667d0d536e80f90c | |
parent | bd65431ca39696ec242d63e4d378bae1a333e0c9 (diff) |
Add interface for an enum store.
52 files changed, 305 insertions, 299 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index a070e907fbb..a1d5f72bc9d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_vector_explorer.h" -#include <vespa/searchlib/attribute/enumstorebase.h> +#include <vespa/searchlib/attribute/i_enum_store.h> #include <vespa/searchlib/attribute/multi_value_mapping.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/ipostinglistattributebase.h> @@ -10,7 +10,7 @@ using search::attribute::Status; using search::AddressSpaceUsage; using search::AttributeVector; -using search::EnumStoreBase; +using search::IEnumStore; using vespalib::AddressSpace; using vespalib::MemoryUsage; using search::attribute::MultiValueMappingBase; @@ -74,7 +74,7 @@ convertMemoryUsageToSlime(const MemoryUsage &usage, Cursor &object) } void -convertEnumStoreToSlime(const EnumStoreBase &enumStore, Cursor &object) +convertEnumStoreToSlime(const IEnumStore &enumStore, Cursor &object) { object.setLong("numUniques", enumStore.getNumUniques()); convertMemoryUsageToSlime(enumStore.getMemoryUsage(), object.setObject("memoryUsage")); @@ -119,7 +119,7 @@ AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bo convertStatusToSlime(status, object.setObject("status")); convertGenerationToSlime(attr, object.setObject("generation")); convertAddressSpaceUsageToSlime(attr.getAddressSpaceUsage(), object.setObject("addressSpaceUsage")); - const EnumStoreBase *enumStore = attr.getEnumStoreBase(); + const IEnumStore *enumStore = attr.getEnumStoreBase(); if (enumStore) { convertEnumStoreToSlime(*enumStore, object.setObject("enumStore")); } diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 833cddb91a7..bd1cfd45280 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1638,9 +1638,9 @@ AttributeTest::testStatus() fillString(values, 16); uint32_t numDocs = 100; // No posting list - static constexpr size_t LeafNodeSize = 4 + sizeof(EnumStoreBase::Index) * EnumTreeTraits::LEAF_SLOTS; + static constexpr size_t LeafNodeSize = 4 + sizeof(IEnumStore::Index) * EnumTreeTraits::LEAF_SLOTS; static constexpr size_t InternalNodeSize = - 8 + (sizeof(EnumStoreBase::Index) + sizeof(datastore::EntryRef)) * EnumTreeTraits::INTERNAL_SLOTS; + 8 + (sizeof(IEnumStore::Index) + sizeof(datastore::EntryRef)) * EnumTreeTraits::INTERNAL_SLOTS; static constexpr size_t NestedVectorSize = 24; // sizeof(vespalib::Array) { @@ -1683,7 +1683,7 @@ AttributeTest::testStatus() expUsed += numUniq * 32; // enum store (16 unique values, 32 bytes per entry) // multi value mapping (numdocs * sizeof(MappingIndex) + numvalues * sizeof(EnumIndex) + // numdocs * sizeof(Array<EnumIndex>) (due to vector vector)) - expUsed += numDocs * sizeof(datastore::EntryRef) + numDocs * numValuesPerDoc * sizeof(EnumStoreBase::Index) + ((numValuesPerDoc > 1024) ? numDocs * NestedVectorSize : 0); + expUsed += numDocs * sizeof(datastore::EntryRef) + numDocs * numValuesPerDoc * sizeof(IEnumStore::Index) + ((numValuesPerDoc > 1024) ? numDocs * NestedVectorSize : 0); EXPECT_GREATER_EQUAL(ptr->getStatus().getUsed(), expUsed); EXPECT_GREATER_EQUAL(ptr->getStatus().getAllocated(), expUsed); } diff --git a/searchlib/src/tests/attribute/comparator/comparator_test.cpp b/searchlib/src/tests/attribute/comparator/comparator_test.cpp index 6ad2a7ebe3b..a2000c48423 100644 --- a/searchlib/src/tests/attribute/comparator/comparator_test.cpp +++ b/searchlib/src/tests/attribute/comparator/comparator_test.cpp @@ -24,7 +24,7 @@ typedef EnumStoreT<StringEntryType> StringEnumStore; typedef EnumStoreComparatorT<StringEntryType> StringComparator; typedef EnumStoreFoldedComparatorT<StringEntryType> FoldedStringComparator; -typedef EnumStoreBase::Index EnumIndex; +typedef IEnumStore::Index EnumIndex; typedef BTreeRoot<EnumIndex, BTreeNoLeafData, btree::NoAggregated, diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index 29c35f11e65..c4ba8eecf43 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -15,7 +15,7 @@ size_t enumStoreAlign(size_t size) return (size + 15) & -UINT64_C(16); } -// EnumStoreBase::Index(0,0) is reserved thus 16 bytes are reserved in buffer 0 +// IEnumStore::Index(0,0) is reserved thus 16 bytes are reserved in buffer 0 const uint32_t RESERVED_BYTES = 16u; typedef EnumStoreT<NumericEntryType<uint32_t> > NumericEnumStore; @@ -26,7 +26,7 @@ private: typedef EnumStoreT<NumericEntryType<float> > FloatEnumStore; typedef EnumStoreT<NumericEntryType<double> > DoubleEnumStore; - typedef EnumStoreBase::Index EnumIndex; + typedef IEnumStore::Index EnumIndex; typedef vespalib::GenerationHandler::generation_t generation_t; void testIndex(); @@ -445,7 +445,7 @@ EnumStoreTest::testCompaction(bool hasPostings) auto old_compaction_count = data_store_base.get_compaction_count(); // perform compaction - EnumStoreBase::EnumIndexMap old2New; + IEnumStore::EnumIndexMap old2New; EXPECT_TRUE(ses.performCompaction(3 * entrySize, old2New)); EXPECT_TRUE(ses.getRemaining() >= 3 * entrySize); EXPECT_TRUE(ses.getBuffer(1).remaining() >= 3 * entrySize); @@ -626,7 +626,7 @@ EnumStoreTest::testHoldListAndGeneration() // perform compaction uint32_t newEntrySize = StringEnumStore::alignEntrySize(8 + 1 + 8); - EnumStoreBase::EnumIndexMap old2New; + IEnumStore::EnumIndexMap old2New; EXPECT_TRUE(ses.performCompaction(5 * newEntrySize, old2New)); // check readers again @@ -715,7 +715,7 @@ EnumStoreTest::testMemoryUsage() EXPECT_EQUAL((num / 2) * entrySize + RESERVED_BYTES, usage.deadBytes()); EXPECT_EQUAL(0u, usage.allocatedBytesOnHold()); - EnumStoreBase::EnumIndexMap old2New; + IEnumStore::EnumIndexMap old2New; ses.performCompaction(400, old2New); // usage after compaction diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index 2d91ac7c689..505394b20ce 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -88,11 +88,11 @@ private: Int32PostingListAttribute; typedef MultiValueNumericPostingAttribute< EnumAttribute<IntegerAttributeTemplate<int32_t> >, - multivalue::Value<EnumStoreBase::Index> > + multivalue::Value<IEnumStore::Index> > Int32ArrayPostingListAttribute; typedef MultiValueNumericPostingAttribute< EnumAttribute<IntegerAttributeTemplate<int32_t> >, - multivalue::WeightedValue<EnumStoreBase::Index> > + multivalue::WeightedValue<IEnumStore::Index> > Int32WsetPostingListAttribute; typedef SingleValueNumericPostingAttribute< @@ -100,11 +100,11 @@ private: FloatPostingListAttribute; typedef MultiValueNumericPostingAttribute< EnumAttribute<FloatingPointAttributeTemplate<float> >, - multivalue::Value<EnumStoreBase::Index> > + multivalue::Value<IEnumStore::Index> > FloatArrayPostingListAttribute; typedef MultiValueNumericPostingAttribute< EnumAttribute<FloatingPointAttributeTemplate<float> >, - multivalue::WeightedValue<EnumStoreBase::Index> > + multivalue::WeightedValue<IEnumStore::Index> > FloatWsetPostingListAttribute; typedef SingleValueStringPostingAttribute StringPostingListAttribute; diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index b7e036017ea..dcf246033ba 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -330,12 +330,12 @@ void StringAttributeTest::testDefaultValueOnAddDoc(AttributeVector & v) EXPECT_EQUAL(0u, v.getNumDocs()); v.addReservedDoc(); EXPECT_EQUAL(1u, v.getNumDocs()); - EXPECT_TRUE( EnumStoreBase::Index(EntryRef(v.getEnum(0))).valid() ); + EXPECT_TRUE( IEnumStore::Index(EntryRef(v.getEnum(0))).valid() ); uint32_t doc(7); EXPECT_TRUE( v.addDoc(doc) ); EXPECT_EQUAL(1u, doc); EXPECT_EQUAL(2u, v.getNumDocs()); - EXPECT_TRUE( EnumStoreBase::Index(EntryRef(v.getEnum(doc))).valid() ); + EXPECT_TRUE( IEnumStore::Index(EntryRef(v.getEnum(doc))).valid() ); EXPECT_EQUAL(0u, strlen(v.getString(doc, NULL, 0))); } @@ -357,7 +357,7 @@ StringAttributeTest::testSingleValue(Attribute & svsa, Config &cfg) EXPECT_TRUE( doc == i ); EXPECT_TRUE( v.getNumDocs() == i + 1 ); EXPECT_TRUE( v.getValueCount(doc) == 1 ); - EXPECT_TRUE( ! EnumStoreBase::Index(EntryRef(v.getEnum(doc))).valid() ); + EXPECT_TRUE( ! IEnumStore::Index(EntryRef(v.getEnum(doc))).valid() ); } std::map<vespalib::string, uint32_t> enums; @@ -366,7 +366,7 @@ StringAttributeTest::testSingleValue(Attribute & svsa, Config &cfg) sprintf(tmp, "enum%u", i % 10); EXPECT_TRUE( v.update(i, tmp) ); EXPECT_TRUE( v.getValueCount(i) == 1 ); - EXPECT_TRUE( ! EnumStoreBase::Index(EntryRef(v.getEnum(i))).valid() ); + EXPECT_TRUE( ! IEnumStore::Index(EntryRef(v.getEnum(i))).valid() ); if ((i % 10) == 9) { v.commit(); for (uint32_t j = i - 9; j <= i; ++j) { diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 3cefb665181..5d76aaf90f8 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -256,7 +256,7 @@ AttributeVector::headerTypeOK(const vespalib::GenericHeader &header) const void AttributeVector::removeOldGenerations(generation_t firstUsed) { (void) firstUsed; } void AttributeVector::onGenerationChange(generation_t generation) { (void) generation; } -const EnumStoreBase * AttributeVector::getEnumStoreBase() const { return nullptr; } +const IEnumStore * AttributeVector::getEnumStoreBase() const { return nullptr; } const attribute::MultiValueMappingBase * AttributeVector::getMultiValueBase() const { return nullptr; } std::unique_ptr<FastOS_FileInterface> diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 20803ca2387..52e63385c7d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -43,7 +43,7 @@ namespace search { template <typename T> class ComponentGuard; class AttributeReadGuard; class AttributeSaver; - class EnumStoreBase; + class IEnumStore; class IAttributeSaveTarget; struct IDocumentWeightAttribute; class QueryTermSimple; @@ -550,7 +550,7 @@ public: attribute::ISearchContext::UP createSearchContext(QueryTermSimpleUP term, const attribute::SearchContextParams ¶ms) const override; virtual SearchContext::UP getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams ¶ms) const = 0; - virtual const EnumStoreBase *getEnumStoreBase() const; + virtual const IEnumStore *getEnumStoreBase() const; virtual const attribute::MultiValueMappingBase *getMultiValueBase() const; private: diff --git a/searchlib/src/vespa/searchlib/attribute/defines.h b/searchlib/src/vespa/searchlib/attribute/defines.h index 649811eded2..1a7964e057b 100644 --- a/searchlib/src/vespa/searchlib/attribute/defines.h +++ b/searchlib/src/vespa/searchlib/attribute/defines.h @@ -5,7 +5,7 @@ #define ENUM_ATTRIBUTE(B) EnumAttribute<B> #define MULTIVALUE_ARG(T) multivalue::Value<T> -#define MULTIVALUE_ENUM_ARG multivalue::Value<EnumStoreBase::Index> +#define MULTIVALUE_ENUM_ARG multivalue::Value<IEnumStore::Index> #define WEIGHTED_MULTIVALUE_ARG(T) multivalue::WeightedValue<T> -#define WEIGHTED_MULTIVALUE_ENUM_ARG multivalue::WeightedValue<EnumStoreBase::Index> +#define WEIGHTED_MULTIVALUE_ENUM_ARG multivalue::WeightedValue<IEnumStore::Index> diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index 690b267ec0c..689aa102d3a 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -20,7 +20,7 @@ namespace search { using btree::BTreeNode; template <typename DictionaryT> -EnumStoreDictionary<DictionaryT>::EnumStoreDictionary(EnumStoreBase& enumStore) +EnumStoreDictionary<DictionaryT>::EnumStoreDictionary(IEnumStore& enumStore) : ParentUniqueStoreDictionary(), _enumStore(enumStore) { @@ -165,10 +165,10 @@ EnumStoreDictionary<DictionaryT>::findFrozenIndex(const datastore::EntryComparat } template <typename DictionaryT> -std::vector<EnumStoreBase::EnumHandle> +std::vector<IEnumStore::EnumHandle> EnumStoreDictionary<DictionaryT>::findMatchingEnums(const datastore::EntryComparator& cmp) const { - std::vector<EnumStoreBase::EnumHandle> result; + std::vector<IEnumStore::EnumHandle> result; auto itr = this->_dict.getFrozenView().find(Index(), cmp); while (itr.valid() && !cmp(Index(), itr.getKey())) { result.push_back(itr.getKey().ref()); @@ -225,94 +225,94 @@ template class EnumStoreDictionary<EnumTree>; template class EnumStoreDictionary<EnumPostingTree>; template -class btree::BTreeNodeT<EnumStoreBase::Index, EnumTreeTraits::INTERNAL_SLOTS>; +class btree::BTreeNodeT<IEnumStore::Index, EnumTreeTraits::INTERNAL_SLOTS>; template -class btree::BTreeNodeTT<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; +class btree::BTreeNodeTT<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; template -class btree::BTreeNodeTT<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeNodeTT<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeInternalNode<EnumStoreBase::Index, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; +class btree::BTreeInternalNode<IEnumStore::Index, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; template -class btree::BTreeLeafNode<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNode<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeLeafNode<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNode<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeLeafNodeTemp<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNodeTemp<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeLeafNodeTemp<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNodeTemp<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeNodeStore<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeNodeStore<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeNodeStore<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeNodeStore<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeRoot<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeRoot<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTreeRoot<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeRoot<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTreeRootT<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeRootT<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTreeRootT<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeRootT<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTreeRootBase<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeRootBase<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeRootBase<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeRootBase<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeNodeAllocator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeNodeAllocator<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeNodeAllocator<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeNodeAllocator<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeIteratorBase<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeIteratorBase<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; template -class btree::BTreeIteratorBase<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeIteratorBase<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; -template class btree::BTreeConstIterator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +template class btree::BTreeConstIterator<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; -template class btree::BTreeConstIterator<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +template class btree::BTreeConstIterator<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTreeIterator<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeIterator<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTreeIterator<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeIterator<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTree<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTree<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; template -class btree::BTree<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTree<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index cd28f10a3cd..31f52e4e965 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -7,7 +7,7 @@ namespace search { -class EnumStoreBase; +class IEnumStore; /** * Concrete dictionary for an enum store that extends the functionality of a unique store dictionary. @@ -22,10 +22,10 @@ private: using ParentUniqueStoreDictionary = datastore::UniqueStoreDictionary<DictionaryT, IEnumStoreDictionary>; using generation_t = IEnumStoreDictionary::generation_t; - EnumStoreBase& _enumStore; + IEnumStore& _enumStore; public: - EnumStoreDictionary(EnumStoreBase& enumStore); + EnumStoreDictionary(IEnumStore& enumStore); ~EnumStoreDictionary() override; @@ -63,95 +63,95 @@ public: }; extern template -class btree::BTreeNodeT<EnumStoreIndex, EnumTreeTraits::INTERNAL_SLOTS>; +class btree::BTreeNodeT<IEnumStore::Index, EnumTreeTraits::INTERNAL_SLOTS>; extern template -class btree::BTreeNodeTT<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; +class btree::BTreeNodeTT<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; extern template -class btree::BTreeNodeTT<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeNodeTT<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeInternalNode<EnumStoreIndex, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; +class btree::BTreeInternalNode<IEnumStore::Index, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS>; extern template -class btree::BTreeLeafNode<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNode<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeLeafNode<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNode<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeLeafNodeTemp<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNodeTemp<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeLeafNodeTemp<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; +class btree::BTreeLeafNodeTemp<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeNodeStore<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeNodeStore<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeNodeStore<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeNodeStore<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeRoot<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeRoot<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTreeRoot<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeRoot<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTreeRootT<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeRootT<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTreeRootT<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeRootT<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTreeRootBase<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeRootBase<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeRootBase<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeRootBase<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeNodeAllocator<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeNodeAllocator<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeNodeAllocator<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeNodeAllocator<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeIteratorBase<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeIteratorBase<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; extern template -class btree::BTreeIteratorBase<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeIteratorBase<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS, EnumTreeTraits::PATH_SIZE>; -extern template class btree::BTreeConstIterator<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +extern template class btree::BTreeConstIterator<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; -extern template class btree::BTreeConstIterator<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +extern template class btree::BTreeConstIterator<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTreeIterator<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeIterator<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTreeIterator<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeIterator<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTree<EnumStoreIndex, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTree<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; extern template -class btree::BTree<EnumStoreIndex, datastore::EntryRef, btree::NoAggregated, +class btree::BTree<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index 0c8ef800648..0bc3d717509 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -51,15 +51,15 @@ public: using EnumStoreBatchUpdater = typename EnumStore::BatchUpdater; protected: - using EnumIndex = EnumStoreBase::Index; - using EnumIndexMap = EnumStoreBase::EnumIndexMap; + using EnumIndex = IEnumStore::Index; + using EnumIndexMap = IEnumStore::EnumIndexMap; EnumStore _enumStore; EnumStore & getEnumStore() { return _enumStore; } const EnumStore & getEnumStore() const { return _enumStore; } - const EnumStoreBase * getEnumStoreBase() const override { return &_enumStore; } + const IEnumStore* getEnumStoreBase() const override { return &_enumStore; } EnumType getFromEnum(EnumHandle e) const override { return _enumStore.getValue(e); } void fillPostings(LoadedVector & loaded) override { (void) loaded; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp index 75cd504faad..1c42eaa5835 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp @@ -8,7 +8,7 @@ namespace search { EnumAttributeSaver:: -EnumAttributeSaver(const EnumStoreBase &enumStore) +EnumAttributeSaver(const IEnumStore &enumStore) : _enumStore(enumStore), _enumerator(_enumStore.getEnumStoreDict(), _enumStore.get_data_store_base()) { @@ -34,6 +34,6 @@ EnumAttributeSaver::writeUdat(IAttributeSaveTarget &saveTarget) namespace search::datastore { -template class UniqueStoreEnumerator<EnumStoreIndex>; +template class UniqueStoreEnumerator<IEnumStore::Index>; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h index 9c703c4f72c..8d0e69afddc 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h @@ -9,7 +9,7 @@ namespace search { class IAttributeSaveTarget; -/* +/** * Helper class for saving an enumerated multivalue attribute. * * It handles writing to the udat file. @@ -17,18 +17,18 @@ class IAttributeSaveTarget; class EnumAttributeSaver { public: - using Enumerator = datastore::UniqueStoreEnumerator<EnumStoreIndex>; + using Enumerator = datastore::UniqueStoreEnumerator<IEnumStore::Index>; private: - const EnumStoreBase &_enumStore; + const IEnumStore &_enumStore; Enumerator _enumerator; public: - EnumAttributeSaver(const EnumStoreBase &enumStore); + EnumAttributeSaver(const IEnumStore &enumStore); ~EnumAttributeSaver(); void writeUdat(IAttributeSaveTarget &saveTarget); - const EnumStoreBase &getEnumStore() const { return _enumStore; } + const IEnumStore &getEnumStore() const { return _enumStore; } Enumerator &get_enumerator() { return _enumerator; } void clear() { _enumerator.clear(); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index de695532f42..c45486f9513 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -83,11 +83,11 @@ EnumStoreT<StringEntryType>::deserialize(const void *src, } template -class btree::BTreeBuilder<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeBuilder<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template -class btree::BTreeBuilder<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeBuilder<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; template class EnumStoreT< StringEntryType >; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 56bf257b046..be2a8eafd9e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -103,7 +103,7 @@ private: } protected: - typedef EnumStoreBase::IndexSet IndexSet; + typedef IEnumStore::IndexSet IndexSet; using EnumStoreBase::_store; using EnumStoreBase::TYPE_ID; @@ -195,8 +195,8 @@ public: ssize_t deserialize(const void *src, size_t available, size_t &initSpace) override; ssize_t deserialize(const void *src, size_t available, Index &idx) override; bool foldedChange(const Index &idx1, const Index &idx2) override; - virtual bool findEnum(Type value, EnumStoreBase::EnumHandle &e) const; - virtual std::vector<EnumStoreBase::EnumHandle> findFoldedEnums(Type value) const; + virtual bool findEnum(Type value, IEnumStore::EnumHandle &e) const; + virtual std::vector<IEnumStore::EnumHandle> findFoldedEnums(Type value) const; void addEnum(Type value, Index &newIdx); virtual bool findIndex(Type value, Index &idx) const; void freeUnusedEnums(bool movePostingidx) override; @@ -263,10 +263,10 @@ insertEntryValue(char * dst, Type value); extern template -class btree::BTreeBuilder<EnumStoreBase::Index, btree::BTreeNoLeafData, btree::NoAggregated, +class btree::BTreeBuilder<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template -class btree::BTreeBuilder<EnumStoreBase::Index, datastore::EntryRef, btree::NoAggregated, +class btree::BTreeBuilder<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, EnumTreeTraits::INTERNAL_SLOTS, EnumTreeTraits::LEAF_SLOTS>; extern template class EnumStoreT< StringEntryType >; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index 4627c7bc03e..297a5e3f807 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -142,7 +142,7 @@ EnumStoreT<EntryType>::foldedChange(const Index &idx1, const Index &idx2) template <typename EntryType> bool -EnumStoreT<EntryType>::findEnum(Type value, EnumStoreBase::EnumHandle &e) const +EnumStoreT<EntryType>::findEnum(Type value, IEnumStore::EnumHandle &e) const { ComparatorType cmp(*this, value); Index idx; @@ -154,7 +154,7 @@ EnumStoreT<EntryType>::findEnum(Type value, EnumStoreBase::EnumHandle &e) const } template <typename EntryType> -std::vector<EnumStoreBase::EnumHandle> +std::vector<IEnumStore::EnumHandle> EnumStoreT<EntryType>::findFoldedEnums(Type value) const { FoldedComparatorType cmp(*this, value); @@ -283,7 +283,7 @@ EnumStoreT<EntryType>::addEnum(Type value, Index & newIdx) template <typename DictionaryType> struct TreeBuilderInserter { static void insert(typename DictionaryType::Builder & builder, - EnumStoreBase::Index enumIdx, + IEnumStore::Index enumIdx, datastore::EntryRef postingIdx) { (void) postingIdx; @@ -294,7 +294,7 @@ struct TreeBuilderInserter { template <> struct TreeBuilderInserter<EnumPostingTree> { static void insert(EnumPostingTree::Builder & builder, - EnumStoreBase::Index enumIdx, + IEnumStore::Index enumIdx, datastore::EntryRef postingIdx) { builder.insert(enumIdx, postingIdx); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp index 71b55689048..7aba3547339 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp @@ -207,73 +207,19 @@ EnumStoreBase::deserialize0(const void *src, } -template <typename Tree> -ssize_t -EnumStoreBase::deserialize(const void *src, - size_t available, - IndexVector &idx, - Tree &tree) -{ - ssize_t sz(deserialize0(src, available, idx)); - if (sz >= 0) { - typename Tree::Builder builder(tree.getAllocator()); - typedef IndexVector::const_iterator IT; - for (IT i(idx.begin()), ie(idx.end()); i != ie; ++i) { - builder.insert(*i, typename Tree::DataType()); - } - tree.assign(builder); - } - return sz; -} - - -template <typename Tree> -void -EnumStoreBase::fixupRefCounts(const EnumVector &hist, Tree &tree) -{ - if ( hist.empty() ) - return; - typename Tree::Iterator ti(tree.begin()); - typedef EnumVector::const_iterator HistIT; - - for (HistIT hi(hist.begin()), hie(hist.end()); hi != hie; ++hi, ++ti) { - assert(ti.valid()); - fixupRefCount(ti.getKey(), *hi); - } - assert(!ti.valid()); - freeUnusedEnums(false); -} - - -vespalib::asciistream & operator << (vespalib::asciistream & os, const EnumStoreBase::Index & idx) { +vespalib::asciistream & operator << (vespalib::asciistream & os, const IEnumStore::Index & idx) { return os << "offset(" << idx.offset() << "), bufferId(" << idx.bufferId() << "), idx(" << idx.ref() << ")"; } -template class datastore::DataStoreT<EnumStoreIndex>; - -template -ssize_t -EnumStoreBase::deserialize<EnumTree>(const void *src, size_t available, IndexVector &idx, EnumTree &tree); - -template -ssize_t -EnumStoreBase::deserialize<EnumPostingTree>(const void *src, size_t available, IndexVector &idx, EnumPostingTree &tree); - -template -void -EnumStoreBase::fixupRefCounts<EnumTree>(const EnumVector &hist, EnumTree &tree); - -template -void -EnumStoreBase::fixupRefCounts<EnumPostingTree>(const EnumVector &hist, EnumPostingTree &tree); +template class datastore::DataStoreT<IEnumStore::Index>; } namespace vespalib { -template class RcuVectorBase<search::EnumStoreIndex>; +template class RcuVectorBase<search::IEnumStore::Index>; } -VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(search::EnumStoreIndex, search::EnumStoreIndex, - vespalib::hash<search::EnumStoreIndex>, std::equal_to<search::EnumStoreIndex>, +VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(search::IEnumStore::Index, search::IEnumStore::Index, + vespalib::hash<search::IEnumStore::Index>, std::equal_to<search::IEnumStore::Index>, vespalib::hashtable_base::and_modulator); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h index 32e7e5134ca..cec6c5e5ac8 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h @@ -6,7 +6,6 @@ #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/vespalib/datastore/datastore.h> #include <vespa/vespalib/datastore/entryref.h> -#include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/address_space.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/memoryusage.h> @@ -23,18 +22,11 @@ namespace attribute { class Status; } using EnumStoreComparator = datastore::EntryComparator; -class EnumStoreBase -{ +class EnumStoreBase : public IEnumStore { public: - using DataStoreType = datastore::DataStoreT<EnumStoreIndex>; - using EnumHandle = attribute::IAttributeVector::EnumHandle; - using EnumVector = EnumStoreEnumVector; - using Index = EnumStoreIndex; - using IndexVector = EnumStoreIndexVector; + using DataStoreType = datastore::DataStoreT<Index>; using generation_t = vespalib::GenerationHandler::generation_t; - using EnumIndexMap = vespalib::hash_map<Index, Index, vespalib::hash<Index>, std::equal_to<Index>, - vespalib::hashtable_base::and_modulator>; class EntryBase { protected: @@ -64,8 +56,6 @@ public: static uint32_t size() { return 2*sizeof(uint32_t); } }; - using IndexSet = std::set<Index, CompareEnumIndex>; - protected: class EnumBufferType : public datastore::BufferType<char> { @@ -131,14 +121,11 @@ public: void decRefCount(Index idx) { getEntryBase(idx).decRefCount(); } // Only use when reading from enumerated attribute save files - void fixupRefCount(Index idx, uint32_t refCount) { + void fixupRefCount(Index idx, uint32_t refCount) override { getEntryBase(idx).setRefCount(refCount); } - template <typename Tree> - void fixupRefCounts(const EnumVector &hist, Tree &tree); - - uint32_t getNumUniques() const { return _enumDict->getNumUniques(); } + uint32_t getNumUniques() const override { return _enumDict->getNumUniques(); } uint32_t getRemaining() const { return _store.getBufferState(_store.getActiveBufferId(TYPE_ID)).remaining(); @@ -146,8 +133,8 @@ public: uint32_t getCapacity() const { return _store.getBufferState(_store.getActiveBufferId(TYPE_ID)).capacity(); } - vespalib::MemoryUsage getMemoryUsage() const; - vespalib::MemoryUsage getTreeMemoryUsage() const { return _enumDict->get_memory_usage(); } + vespalib::MemoryUsage getMemoryUsage() const override; + vespalib::MemoryUsage getTreeMemoryUsage() const override { return _enumDict->get_memory_usage(); } vespalib::AddressSpace getAddressSpaceUsage() const; @@ -164,23 +151,15 @@ public: bool getPendingCompact() const { return _type.getPendingCompact(); } void clearPendingCompact() { _type.clearPendingCompact(); } - virtual void writeValues(BufferWriter &writer, const Index *idxs, size_t count) const = 0; - virtual ssize_t deserialize(const void *src, size_t available, size_t &initSpace) = 0; virtual ssize_t deserialize(const void *src, size_t available, Index &idx) = 0; - virtual bool foldedChange(const Index &idx1, const Index &idx2) = 0; - - ssize_t deserialize0(const void *src, size_t available, IndexVector &idx); - template <typename Tree> - ssize_t deserialize(const void *src, size_t available, IndexVector &idx, Tree &tree); + ssize_t deserialize0(const void *src, size_t available, IndexVector &idx) override; ssize_t deserialize(const void *src, size_t available, IndexVector &idx) { return _enumDict->deserialize(src, available, idx); } - virtual void freeUnusedEnum(Index idx, IndexSet &unused) = 0; - virtual void freeUnusedEnums(bool movePostingIdx) = 0; virtual void freeUnusedEnums(const IndexSet& toRemove) = 0; void fixupRefCounts(const EnumVector &hist) { _enumDict->fixupRefCounts(hist); } @@ -188,20 +167,20 @@ public: virtual bool performCompaction(uint64_t bytesNeeded, EnumIndexMap & old2New) = 0; - IEnumStoreDictionary &getEnumStoreDict() { return *_enumDict; } - const IEnumStoreDictionary &getEnumStoreDict() const { return *_enumDict; } + IEnumStoreDictionary &getEnumStoreDict() override { return *_enumDict; } + const IEnumStoreDictionary &getEnumStoreDict() const override { return *_enumDict; } EnumPostingTree &getPostingDictionary() { return _enumDict->getPostingDictionary(); } const EnumPostingTree &getPostingDictionary() const { return _enumDict->getPostingDictionary(); } - const datastore::DataStoreBase &get_data_store_base() const { return _store; } + const datastore::DataStoreBase &get_data_store_base() const override { return _store; } }; -vespalib::asciistream & operator << (vespalib::asciistream & os, const EnumStoreBase::Index & idx); +vespalib::asciistream & operator << (vespalib::asciistream & os, const IEnumStore::Index & idx); extern template -class datastore::DataStoreT<EnumStoreIndex>; +class datastore::DataStoreT<IEnumStore::Index>; } diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp index a454fd3e235..3925bb3dd4e 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp @@ -48,7 +48,7 @@ FloatingPointAttributeTemplate<T>::findEnum(const char *value, EnumHandle &e) co } template<typename T> -std::vector<EnumStoreBase::EnumHandle> +std::vector<IEnumStore::EnumHandle> FloatingPointAttributeTemplate<T>::findFoldedEnums(const char *value) const { std::vector<EnumHandle> result; diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h new file mode 100644 index 00000000000..0963e0ff67d --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -0,0 +1,93 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/vespalib/datastore/entryref.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/memoryusage.h> +#include <cassert> +#include <set> + +namespace search { + +namespace datastore { class DataStoreBase; } + +class BufferWriter; +class IEnumStoreDictionary; + +/** + * Interface for an enum store that is independent of the data type stored. + */ +class IEnumStore { +public: + using Index = datastore::AlignedEntryRefT<31, 4>; + using IndexVector = vespalib::Array<Index>; + using EnumHandle = attribute::IAttributeVector::EnumHandle; + using EnumVector = vespalib::Array<uint32_t>; + + using EnumIndexMap = vespalib::hash_map<Index, Index, vespalib::hash<Index>, std::equal_to<Index>, + vespalib::hashtable_base::and_modulator>; + + struct CompareEnumIndex { + using Index = IEnumStore::Index; + + bool operator()(const Index &lhs, const Index &rhs) const { + return lhs.ref() < rhs.ref(); + } + }; + + using IndexSet = std::set<Index, CompareEnumIndex>; + + virtual ~IEnumStore() = default; + + virtual void writeValues(BufferWriter& writer, const Index* idxs, size_t count) const = 0; + virtual ssize_t deserialize0(const void* src, size_t available, IndexVector& idx) = 0; + virtual void fixupRefCount(Index idx, uint32_t refCount) = 0; + virtual void freeUnusedEnum(Index idx, IndexSet& unused) = 0; + virtual void freeUnusedEnums(bool movePostingIdx) = 0; + virtual bool foldedChange(const Index& idx1, const Index& idx2) = 0; + virtual IEnumStoreDictionary& getEnumStoreDict() = 0; + virtual const IEnumStoreDictionary& getEnumStoreDict() const = 0; + virtual const datastore::DataStoreBase& get_data_store_base() const = 0; + virtual uint32_t getNumUniques() const = 0; + virtual vespalib::MemoryUsage getMemoryUsage() const = 0; + virtual vespalib::MemoryUsage getTreeMemoryUsage() const = 0; + + + template <typename TreeT> + ssize_t deserialize(const void *src, + size_t available, + IndexVector& idx, + TreeT& tree) { + ssize_t sz(deserialize0(src, available, idx)); + if (sz >= 0) { + typename TreeT::Builder builder(tree.getAllocator()); + typedef IndexVector::const_iterator IT; + for (IT i(idx.begin()), ie(idx.end()); i != ie; ++i) { + builder.insert(*i, typename TreeT::DataType()); + } + tree.assign(builder); + } + return sz; + } + + template <typename TreeT> + void fixupRefCounts(const EnumVector& hist, TreeT& tree) { + if (hist.empty()) { + return; + } + typename TreeT::Iterator ti(tree.begin()); + typedef EnumVector::const_iterator HistIT; + + for (HistIT hi(hist.begin()), hie(hist.end()); hi != hie; ++hi, ++ti) { + assert(ti.valid()); + fixupRefCount(ti.getKey(), *hi); + } + assert(!ti.valid()); + freeUnusedEnums(false); + } +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h index 3c324a30779..a4c14256835 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store_dictionary.h @@ -2,48 +2,36 @@ #pragma once +#include "i_enum_store.h" #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/vespalib/datastore/entry_comparator_wrapper.h> #include <vespa/vespalib/datastore/unique_store_dictionary.h> -#include <set> namespace search { class BufferWriter; -using EnumStoreIndex = datastore::AlignedEntryRefT<31, 4>; -using EnumStoreIndexVector = vespalib::Array<EnumStoreIndex>; -using EnumStoreEnumVector = vespalib::Array<uint32_t>; - using EnumTreeTraits = btree::BTreeTraits<16, 16, 10, true>; -using EnumTree = btree::BTree<EnumStoreIndex, btree::BTreeNoLeafData, +using EnumTree = btree::BTree<IEnumStore::Index, btree::BTreeNoLeafData, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; -using EnumPostingTree = btree::BTree<EnumStoreIndex, datastore::EntryRef, +using EnumPostingTree = btree::BTree<IEnumStore::Index, datastore::EntryRef, btree::NoAggregated, const datastore::EntryComparatorWrapper, EnumTreeTraits>; -struct CompareEnumIndex { - using Index = EnumStoreIndex; - - bool operator()(const Index &lhs, const Index &rhs) const { - return lhs.ref() < rhs.ref(); - } -}; - /** * Interface for the dictionary used by an enum store. */ class IEnumStoreDictionary : public datastore::IUniqueStoreDictionary { public: - using EnumVector = EnumStoreEnumVector; - using Index = EnumStoreIndex; - using IndexSet = std::set<Index, CompareEnumIndex>; - using IndexVector = EnumStoreIndexVector; + using EnumVector = IEnumStore::EnumVector; + using Index = IEnumStore::Index; + using IndexSet = IEnumStore::IndexSet; + using IndexVector = IEnumStore::IndexVector; using generation_t = vespalib::GenerationHandler::generation_t; public: diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp index 7b23cd51e92..a08ce394a46 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp @@ -34,7 +34,7 @@ IntegerAttributeTemplate<T>::findEnum(const char *value, EnumHandle &e) const { template<typename T> -std::vector<EnumStoreBase::EnumHandle> +std::vector<IEnumStore::EnumHandle> IntegerAttributeTemplate<T>::findFoldedEnums(const char *value) const { std::vector<EnumHandle> result; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp index 96df7d06b39..b33d604c4a8 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp @@ -26,7 +26,7 @@ INSTANTIATE_ARRAY(ValueType, Saver); \ INSTANTIATE_WSET(ValueType, Saver) #define INSTANTIATE_ENUM(Saver) \ -INSTANTIATE_SINGLE_ARRAY_WSET(EnumStoreIndex, Saver) +INSTANTIATE_SINGLE_ARRAY_WSET(IEnumStore::Index, Saver) #define INSTANTIATE_VALUE(ValueType) \ INSTANTIATE_SINGLE_ARRAY_WSET(ValueType, NoSaveLoadedEnum) diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h index 49cadf379db..a58fa479249 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h +++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h @@ -125,7 +125,7 @@ class SaveEnumHist vespalib::ArrayRef<uint32_t> _hist; public: - SaveEnumHist(EnumStoreBase::EnumVector &enumHist) + SaveEnumHist(IEnumStore::EnumVector &enumHist) : _hist(enumHist) { } diff --git a/searchlib/src/vespa/searchlib/attribute/loadedvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedvalue.h index 38baed6b3ea..f8af21e6fa7 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedvalue.h +++ b/searchlib/src/vespa/searchlib/attribute/loadedvalue.h @@ -57,14 +57,14 @@ public: } }; - EnumStoreBase::Index + IEnumStore::Index getEidx() const { - return EnumStoreBase::Index(datastore::EntryRef(_value._eidx)); + return IEnumStore::Index(datastore::EntryRef(_value._eidx)); } void - setEidx(EnumStoreBase::Index v) + setEidx(IEnumStore::Index v) { _value._eidx = v.ref(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp index ab047c41e2d..8cc6d6d6ac6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.cpp @@ -13,8 +13,8 @@ using search::multivalue::WeightedValue; namespace search { namespace attribute { -template class MultiValueMapping<Value<EnumStoreIndex>>; -template class MultiValueMapping<WeightedValue<EnumStoreIndex>>; +template class MultiValueMapping<Value<IEnumStore::Index>>; +template class MultiValueMapping<WeightedValue<IEnumStore::Index>>; template class MultiValueMapping<Value<int8_t>>; template class MultiValueMapping<WeightedValue<int8_t>>; template class MultiValueMapping<Value<int16_t>>; diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.h b/searchlib/src/vespa/searchlib/attribute/multienumattribute.h index c0f8e3515d3..31cc0fa2d12 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.h @@ -12,7 +12,7 @@ namespace search { class IWeightedIndexVector { public: virtual ~IWeightedIndexVector() = default; - using WeightedIndex = multivalue::WeightedValue<EnumStoreBase::Index>; + using WeightedIndex = multivalue::WeightedValue<IEnumStore::Index>; /** * Provides a reference to the underlying enum/weight pairs. * This method should only be invoked if @ref getCollectionType(docId) returns CollectionType::WEIGHTED_SET. @@ -49,11 +49,11 @@ protected: using generation_t = typename B::BaseClass::generation_t; using DocIndices = typename MultiValueAttribute<B, M>::DocumentValues; - using EnumIndex = typename EnumStoreBase::Index; - using EnumIndexMap = EnumStoreBase::EnumIndexMap; - using EnumIndexVector = typename EnumStoreBase::IndexVector; + using EnumIndex = IEnumStore::Index; + using EnumIndexMap = IEnumStore::EnumIndexMap; + using EnumIndexVector = IEnumStore::IndexVector; using EnumStoreBatchUpdater = typename B::EnumStoreBatchUpdater; - using EnumVector = typename EnumStoreBase::EnumVector; + using EnumVector = IEnumStore::EnumVector; using LoadedEnumAttribute = attribute::LoadedEnumAttribute; using LoadedEnumAttributeVector = attribute::LoadedEnumAttributeVector; using WeightedIndex = typename MultiValueAttribute<B, M>::MultiValueType; diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp index 32f46d859bc..13079b6591c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp @@ -21,7 +21,7 @@ namespace { */ class DatWriter { - std::vector<EnumStoreIndex> _indexes; + std::vector<IEnumStore::Index> _indexes; const EnumAttributeSaver::Enumerator &_enumerator; std::unique_ptr<search::BufferWriter> _datWriter; std::function<bool()> _compaction_interferred; @@ -80,7 +80,7 @@ MultiValueEnumAttributeSaver<MultiValueT>:: MultiValueEnumAttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header, const MultiValueMapping &mvMapping, - const EnumStoreBase &enumStore) + const IEnumStore &enumStore) : Parent(std::move(guard), header, mvMapping), _mvMapping(mvMapping), _enumSaver(enumStore), @@ -134,8 +134,8 @@ onSave(IAttributeSaveTarget &saveTarget) return !compaction_broke_save; } -using EnumIdxArray = multivalue::Value<EnumStoreIndex>; -using EnumIdxWset = multivalue::WeightedValue<EnumStoreIndex>; +using EnumIdxArray = multivalue::Value<IEnumStore::Index>; +using EnumIdxWset = multivalue::WeightedValue<IEnumStore::Index>; template class MultiValueEnumAttributeSaver<EnumIdxArray>; template class MultiValueEnumAttributeSaver<EnumIdxWset>; diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h index d5d0db7acdc..3f770c2a0bb 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h @@ -7,7 +7,7 @@ namespace search { -/* +/** * Class for saving an enumerated multivalue attribute. * * Template argument MultiValueT is either multivalue::Value<ValueType> or @@ -33,7 +33,7 @@ public: MultiValueEnumAttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header, const MultiValueMapping &mvMapping, - const EnumStoreBase &enumStore); + const IEnumStore &enumStore); ~MultiValueEnumAttributeSaver() override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index 498a1e398f9..75453268120 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -24,9 +24,9 @@ public: protected: using DocId = typename B::BaseClass::DocId; using EnumHandle = typename B::BaseClass::EnumHandle; - using EnumIndex = EnumStoreBase::Index; - using EnumIndexVector = EnumStoreBase::IndexVector; - using EnumVector = EnumStoreBase::EnumVector; + using EnumIndex = IEnumStore::Index; + using EnumIndexVector = IEnumStore::IndexVector; + using EnumVector = IEnumStore::EnumVector; using LoadedEnumAttribute = attribute::LoadedEnumAttribute; using LoadedEnumAttributeVector = attribute::LoadedEnumAttributeVector; using LoadedNumericValueT = typename B::BaseClass::LoadedNumericValueT; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h index e990a1030f5..32e98ef7d6d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h @@ -14,8 +14,8 @@ namespace search { * This class is used for both array and weighted set types. * * B: EnumAttribute<P, BaseClass> - * M: multivalue::Value<EnumStoreBase::Index> (array) or - * multivalue::WeightedValue<EnumStoreBase::Index> (weighted set) + * M: multivalue::Value<IEnumStore::Index> (array) or + * multivalue::WeightedValue<IEnumStore::Index> (weighted set) * M specifies the type stored in the MultiValueMapping */ template <typename B, typename M> diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp index 6c0d9342f39..5c08d1f6184 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.cpp @@ -6,8 +6,8 @@ namespace search { -template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<EnumStoreBase::Index> >; -template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<EnumStoreBase::Index> >; +template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<IEnumStore::Index> >; +template class MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<IEnumStore::Index> >; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index ef5158c162a..c117aeea6b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -19,8 +19,8 @@ namespace search { * This class is used for both array and weighted set types. * * B: Base class: EnumAttribute<StringAttribute> - * M: multivalue::Value<EnumStoreBase::Index> (array) or - * multivalue::WeightedValue<EnumStoreBase::Index> (weighted set) + * M: multivalue::Value<IEnumStore::Index> (array) or + * multivalue::WeightedValue<IEnumStore::Index> (weighted set) */ template <typename B, typename M> class MultiValueStringAttributeT : public MultiValueEnumAttribute<B, M> { @@ -169,7 +169,7 @@ public: }; -using ArrayStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<EnumStoreBase::Index> >; -using WeightedSetStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<EnumStoreBase::Index> >; +using ArrayStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<IEnumStore::Index> >; +using WeightedSetStringAttribute = MultiValueStringAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<IEnumStore::Index> >; } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp index 75e05d61b13..6f0ea37896f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.cpp @@ -8,8 +8,8 @@ LOG_SETUP(".searchlib.attribute.multi_string_post_attribute"); namespace search { -EnumStoreBase::Index -StringEnumIndexMapper::map(EnumStoreBase::Index original, const EnumStoreComparator & compare) const +IEnumStore::Index +StringEnumIndexMapper::map(IEnumStore::Index original, const EnumStoreComparator & compare) const { return _dictionary.find(original, compare).getKey(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index 88df7ec9704..c98e6db4fde 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -14,8 +14,8 @@ namespace search { * This class is used for both array and weighted set types. * * B: EnumAttribute<StringAttribute> - * T: multivalue::Value<EnumStoreBase::Index> (array) or - * multivalue::WeightedValue<EnumStoreBase::Index> (weighted set) + * T: multivalue::Value<IEnumStore::Index> (array) or + * multivalue::WeightedValue<IEnumStore::Index> (weighted set) */ template <typename B, typename T> class MultiValueStringPostingAttributeT @@ -112,8 +112,8 @@ public: } }; -using ArrayStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<EnumStoreBase::Index> >; -using WeightedSetStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<EnumStoreBase::Index> >; +using ArrayStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::Value<IEnumStore::Index> >; +using WeightedSetStringPostingAttribute = MultiValueStringPostingAttributeT<EnumAttribute<StringAttribute>, multivalue::WeightedValue<IEnumStore::Index> >; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 8268d1ace20..8d956b55733 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -30,7 +30,7 @@ class StringEnumIndexMapper : public EnumIndexMapper { public: StringEnumIndexMapper(const EnumPostingTree & dictionary) : _dictionary(dictionary) { } - EnumStoreBase::Index map(EnumStoreBase::Index original, const EnumStoreComparator & compare) const override; + IEnumStore::Index map(IEnumStore::Index original, const EnumStoreComparator & compare) const override; virtual bool hasFold() const override { return true; } private: const EnumPostingTree & _dictionary; diff --git a/searchlib/src/vespa/searchlib/attribute/numericbase.h b/searchlib/src/vespa/searchlib/attribute/numericbase.h index 08a61620071..f05f99a622a 100644 --- a/searchlib/src/vespa/searchlib/attribute/numericbase.h +++ b/searchlib/src/vespa/searchlib/attribute/numericbase.h @@ -14,9 +14,9 @@ class ReaderBase; class NumericAttribute : public AttributeVector { protected: - typedef EnumStoreBase::Index EnumIndex; - typedef EnumStoreBase::IndexVector EnumIndexVector; - typedef EnumStoreBase::EnumVector EnumVector; + typedef IEnumStore::Index EnumIndex; + typedef IEnumStore::IndexVector EnumIndexVector; + typedef IEnumStore::EnumVector EnumVector; NumericAttribute(const vespalib::string & name, const AttributeVector::Config & cfg) : AttributeVector(name, cfg) diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp index 1eb1fc8c3ee..45bcfb0beac 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.cpp @@ -101,8 +101,8 @@ removeDupRemovals(std::vector<uint32_t> &removals) } -EnumStoreBase::Index -EnumIndexMapper::map(EnumStoreBase::Index original, const EnumStoreComparator & compare) const +IEnumStore::Index +EnumIndexMapper::map(IEnumStore::Index original, const EnumStoreComparator & compare) const { (void) compare; return original; @@ -163,7 +163,7 @@ PostingChange<P>::apply(GrowableBitVector &bv) template <typename WeightedIndex> class ActualChangeComputer { public: - using EnumIndex = EnumStoreBase::Index; + using EnumIndex = IEnumStore::Index; using AlwaysWeightedIndexVector = std::vector<multivalue::WeightedValue<EnumIndex>>; using WeightedIndexVector = std::vector<WeightedIndex>; void compute(const WeightedIndex * entriesNew, size_t szNew, @@ -219,7 +219,7 @@ private: template <typename WeightedIndex> class MergeDupIterator { using InnerIter = typename std::vector<WeightedIndex>::const_iterator; - using EnumIndex = EnumStoreBase::Index; + using EnumIndex = IEnumStore::Index; using Entry = multivalue::WeightedValue<EnumIndex>; InnerIter _cur; InnerIter _end; @@ -350,7 +350,7 @@ template class PostingChange<AttributeWeightPosting>; typedef PostingChange<btree::BTreeKeyData<unsigned int, int> > WeightedPostingChange; typedef std::map<EnumPostingPair, WeightedPostingChange> WeightedPostingChangeMap; -typedef EnumStoreBase::Index EnumIndex; +typedef IEnumStore::Index EnumIndex; typedef multivalue::WeightedValue<EnumIndex> WeightedIndex; typedef multivalue::Value<EnumIndex> ValueIndex; diff --git a/searchlib/src/vespa/searchlib/attribute/postingchange.h b/searchlib/src/vespa/searchlib/attribute/postingchange.h index 00904954a2b..3d5cafe098a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingchange.h +++ b/searchlib/src/vespa/searchlib/attribute/postingchange.h @@ -48,7 +48,7 @@ class EnumIndexMapper { public: virtual ~EnumIndexMapper() { } - virtual EnumStoreBase::Index map(EnumStoreBase::Index original, const EnumStoreComparator & compare) const; + virtual IEnumStore::Index map(IEnumStore::Index original, const EnumStoreComparator & compare) const; virtual bool hasFold() const { return false; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index aa515e82c94..29cb36c8e88 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -12,12 +12,12 @@ using attribute::LoadedNumericValue; template <typename P> PostingListAttributeBase<P>:: PostingListAttributeBase(AttributeVector &attr, - EnumStoreBase &enumStore) + IEnumStore &enumStore) : attribute::IPostingListAttributeBase(), - _postingList(enumStore.getPostingDictionary(), attr.getStatus(), + _postingList(enumStore.getEnumStoreDict().getPostingDictionary(), attr.getStatus(), attr.getConfig()), _attr(attr), - _dict(enumStore.getPostingDictionary()), + _dict(enumStore.getEnumStoreDict().getPostingDictionary()), _esb(enumStore) { } @@ -53,7 +53,7 @@ PostingListAttributeBase<P>::fillPostingsFixupEnumBase(const LoadedEnumAttribute { clearAllPostings(); uint32_t docIdLimit = _attr.getNumDocs(); - EnumStoreBase &enumStore = _esb; + IEnumStore &enumStore = _esb; EntryRef newIndex; PostingChange<P> postings; if (loaded.empty()) { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h index b9346daefa2..06c96e07a93 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h @@ -19,16 +19,16 @@ namespace search { class EnumPostingPair { private: - EnumStoreBase::Index _idx; + IEnumStore::Index _idx; const datastore::EntryComparator *_cmp; public: - EnumPostingPair(EnumStoreBase::Index idx, const datastore::EntryComparator *cmp) + EnumPostingPair(IEnumStore::Index idx, const datastore::EntryComparator *cmp) : _idx(idx), _cmp(cmp) { } bool operator<(const EnumPostingPair &rhs) const { return (*_cmp)(_idx, rhs._idx); } - EnumStoreBase::Index getEnumIdx() const { return _idx; } + IEnumStore::Index getEnumIdx() const { return _idx; } }; @@ -41,7 +41,7 @@ protected: using AggregationTraits = attribute::PostingListTraits<DataType>; using DocId = AttributeVector::DocId; using EntryRef = datastore::EntryRef; - using EnumIndex = EnumStoreBase::Index; + using EnumIndex = IEnumStore::Index; using LoadedEnumAttributeVector = attribute::LoadedEnumAttributeVector; using PostingList = typename AggregationTraits::PostingList; using PostingMap = std::map<EnumPostingPair, PostingChange<P> >; @@ -49,9 +49,9 @@ protected: PostingList _postingList; AttributeVector &_attr; EnumPostingTree &_dict; - EnumStoreBase &_esb; + IEnumStore &_esb; - PostingListAttributeBase(AttributeVector &attr, EnumStoreBase &enumStore); + PostingListAttributeBase(AttributeVector &attr, IEnumStore &enumStore); virtual ~PostingListAttributeBase(); virtual void updatePostings(PostingMap & changePost) = 0; @@ -82,7 +82,7 @@ public: using Dictionary = EnumPostingTree; using EntryRef = datastore::EntryRef; - using EnumIndex = EnumStoreBase::Index; + using EnumIndex = IEnumStore::Index; using EnumStore = EnumStoreType; using FoldedComparatorType = typename EnumStore::FoldedComparatorType; using LoadedEnumAttributeVector = attribute::LoadedEnumAttributeVector; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 3c3242ce47c..97d54022cc2 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -15,7 +15,7 @@ PostingListSearchContext(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, - const EnumStoreBase &esb, + const IEnumStore &esb, uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &baseSearchCtx) diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 6a2324cef07..f31fbd8423d 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -26,7 +26,7 @@ protected: using Dictionary = EnumPostingTree; using DictionaryConstIterator = Dictionary::ConstIterator; using FrozenDictionary = Dictionary::FrozenView; - using EnumIndex = EnumStoreBase::Index; + using EnumIndex = IEnumStore::Index; const FrozenDictionary _frozenDictionary; DictionaryConstIterator _lowerDictItr; @@ -41,14 +41,14 @@ protected: datastore::EntryRef _frozenRoot; // Posting list in tree form float _FSTC; // Filtering Search Time Constant float _PLSTC; // Posting List Search Time Constant - const EnumStoreBase &_esb; + const IEnumStore &_esb; uint32_t _minBvDocFreq; const GrowableBitVector *_gbv; // bitvector if _useBitVector has been set const ISearchContext &_baseSearchCtx; PostingListSearchContext(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, - const EnumStoreBase &esb, uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &baseSearchCtx); + const IEnumStore &esb, uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &baseSearchCtx); ~PostingListSearchContext(); @@ -113,7 +113,7 @@ protected: static const long MIN_APPROXHITS_TO_NUMDOCS_RATIO_BEFORE_APPROXIMATION = 10; PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, - bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + bool hasWeight, const PostingList &postingList, const IEnumStore &esb, uint32_t minBvCocFreq, bool useBitVector, const ISearchContext &baseSearchCtx); ~PostingListSearchContextT(); @@ -151,7 +151,7 @@ protected: using Parent::singleHits; PostingListFoldedSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, - bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + bool hasWeight, const PostingList &postingList, const IEnumStore &esb, uint32_t minBvCocFreq, bool useBitVector, const ISearchContext &baseSearchCtx); unsigned int approximateHits() const override; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index 15ff9c1ea7a..e2ac4069172 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -20,7 +20,7 @@ namespace search::attribute { template <typename DataT> PostingListSearchContextT<DataT>:: PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, bool hasWeight, - const PostingList &postingList, const EnumStoreBase &esb, + const PostingList &postingList, const IEnumStore &esb, uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &searchContext) : PostingListSearchContext(dictionary, docIdLimit, numValues, hasWeight, esb, minBvDocFreq, useBitVector, searchContext), _postingList(postingList), @@ -286,7 +286,7 @@ PostingListSearchContextT<DataT>::applyRangeLimit(int rangeLimit) template <typename DataT> PostingListFoldedSearchContextT<DataT>:: PostingListFoldedSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uint64_t numValues, - bool hasWeight, const PostingList &postingList, const EnumStoreBase &esb, + bool hasWeight, const PostingList &postingList, const IEnumStore &esb, uint32_t minBvDocFreq, bool useBitVector, const ISearchContext &searchContext) : Parent(dictionary, docIdLimit, numValues, hasWeight, postingList, esb, minBvDocFreq, useBitVector, searchContext) { diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp index 0e122c9d583..9e5c9f0bc7b 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp @@ -29,7 +29,7 @@ AttributeVector::DocId SingleValueEnumAttributeBase::addDoc(bool &incGeneration) { incGeneration = _enumIndices.isFull(); - _enumIndices.push_back(EnumStoreBase::Index()); + _enumIndices.push_back(IEnumStore::Index()); return _enumIndices.size() - 1; } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h index 0edb12325d2..5624ebe6582 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h @@ -19,14 +19,14 @@ class SingleValueEnumAttributeBase { protected: using DocId = AttributeVector::DocId; using EnumHandle = AttributeVector::EnumHandle; - using EnumIndex = EnumStoreBase::Index; + using EnumIndex = IEnumStore::Index; using EnumIndexVector = vespalib::RcuVectorBase<EnumIndex>; using GenerationHolder = vespalib::GenerationHolder; public: using EnumIndexCopyVector = vespalib::Array<EnumIndex>; - EnumStoreBase::Index getEnumIndex(DocId docId) const { return _enumIndices[docId]; } + IEnumStore::Index getEnumIndex(DocId docId) const { return _enumIndices[docId]; } EnumHandle getE(DocId doc) const { return _enumIndices[doc].ref(); } protected: SingleValueEnumAttributeBase(const attribute::Config & c, GenerationHolder &genHolder); @@ -45,7 +45,7 @@ protected: using ChangeVector = typename B::ChangeVector; using ChangeVectorIterator = typename B::ChangeVector::const_iterator; using DocId = typename B::DocId; - using EnumIndexMap = EnumStoreBase::EnumIndexMap; + using EnumIndexMap = IEnumStore::EnumIndexMap; using EnumModifier = typename B::EnumModifier; using EnumStore = typename B::EnumStore; using EnumStoreBatchUpdater = typename EnumStore::BatchUpdater; @@ -87,12 +87,12 @@ protected: void fillValues(LoadedVector & loaded) override; void fillEnumIdx(ReaderBase &attrReader, - const EnumStoreBase::IndexVector &eidxs, + const IEnumStore::IndexVector &eidxs, LoadedEnumAttributeVector &loaded) override; void fillEnumIdx(ReaderBase &attrReader, - const EnumStoreBase::IndexVector &eidxs, - EnumStoreBase::EnumVector &enumHist) override; + const IEnumStore::IndexVector &eidxs, + IEnumStore::EnumVector &enumHist) override; /** * Called when a new document has been added. diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index fcee2c8635f..60a3a74b873 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -217,7 +217,7 @@ SingleValueEnumAttribute<B>::fillValues(LoadedVector & loaded) template <typename B> void SingleValueEnumAttribute<B>::fillEnumIdx(ReaderBase &attrReader, - const EnumStoreBase::IndexVector &eidxs, + const IEnumStore::IndexVector &eidxs, LoadedEnumAttributeVector &loaded) { attribute::loadFromEnumeratedSingleValue(_enumIndices, @@ -231,8 +231,8 @@ SingleValueEnumAttribute<B>::fillEnumIdx(ReaderBase &attrReader, template <typename B> void SingleValueEnumAttribute<B>::fillEnumIdx(ReaderBase &attrReader, - const EnumStoreBase::IndexVector &eidxs, - EnumStoreBase::EnumVector &enumHist) + const IEnumStore::IndexVector &eidxs, + IEnumStore::EnumVector &enumHist) { attribute::loadFromEnumeratedSingleValue(_enumIndices, getGenerationHolder(), diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp index 31569cc9535..afa893458b4 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp @@ -14,7 +14,7 @@ SingleValueEnumAttributeSaver:: SingleValueEnumAttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header, EnumIndexCopyVector &&indices, - const EnumStoreBase &enumStore) + const IEnumStore &enumStore) : AttributeSaver(std::move(guard), header), _indices(std::move(indices)), _enumSaver(enumStore) diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h index 5a58e75a203..29210a8f31d 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h @@ -24,7 +24,7 @@ public: SingleValueEnumAttributeSaver(vespalib::GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header, EnumIndexCopyVector &&indices, - const EnumStoreBase &enumStore); + const IEnumStore &enumStore); virtual ~SingleValueEnumAttributeSaver(); }; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h index 49f907a66a7..dcf71ff9132 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h @@ -22,10 +22,10 @@ protected: using DocId = typename B::BaseClass::DocId; using EnumHandle = typename B::BaseClass::EnumHandle; using EnumIndex = typename SingleValueEnumAttributeBase::EnumIndex; - using EnumIndexVector = EnumStoreBase::IndexVector; + using EnumIndexVector = IEnumStore::IndexVector; using EnumStore = typename SingleValueEnumAttribute<B>::EnumStore; using EnumStoreBatchUpdater = typename EnumStore::BatchUpdater; - using EnumVector = EnumStoreBase::EnumVector; + using EnumVector = IEnumStore::EnumVector; using LoadedEnumAttribute = attribute::LoadedEnumAttribute; using LoadedEnumAttributeVector = attribute::LoadedEnumAttributeVector; using LoadedNumericValueT = typename B::BaseClass::LoadedNumericValueT; diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index 9839cfe6004..21d36e66cf0 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -22,9 +22,9 @@ class StringAttribute : public AttributeVector public: typedef vespalib::Array<uint32_t> OffsetVector; typedef const char * LoadedValueType; - typedef EnumStoreBase::Index EnumIndex; - typedef EnumStoreBase::IndexVector EnumIndexVector; - typedef EnumStoreBase::EnumVector EnumVector; + typedef IEnumStore::Index EnumIndex; + typedef IEnumStore::IndexVector EnumIndexVector; + typedef IEnumStore::EnumVector EnumVector; typedef attribute::LoadedStringVector LoadedVector; public: DECLARE_IDENTIFIABLE_ABSTRACT(StringAttribute); |