diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-08-26 09:23:19 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-08-26 09:23:19 +0200 |
commit | 20a78655195cae198aba967087f75dea46b77204 (patch) | |
tree | b4957d6e3c725100b45cfaacea93b7848c85cc98 | |
parent | a7cd1ff750feb925a9bb299b46716e18c910f5fd (diff) | |
parent | a314d1bd0a3f53cb26b6f5de02dd5b20bd5bf79f (diff) |
Merge pull request #10398 from vespa-engine/toregge/add-workaround-for-aligned-entry-ref
Rename UniqueStoreSaver to UniqueStoreEnumerator.
-rw-r--r-- | searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp | 6 | ||||
-rw-r--r-- | searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp | 14 | ||||
-rw-r--r-- | searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h | 6 | ||||
-rw-r--r-- | vespalib/src/tests/datastore/unique_store/unique_store_test.cpp | 22 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/datastore/datastore.hpp | 5 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/datastore/entryref.h | 4 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/datastore/unique_store.h | 8 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/datastore/unique_store.hpp | 8 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h (renamed from vespalib/src/vespa/vespalib/datastore/unique_store_saver.h) | 17 | ||||
-rw-r--r-- | vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp (renamed from vespalib/src/vespa/vespalib/datastore/unique_store_saver.hpp) | 24 |
10 files changed, 58 insertions, 56 deletions
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 8b1b272be52..406400cef00 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -55,8 +55,8 @@ ReferenceAttribute::~ReferenceAttribute() _referenceMappings.clearBuilder(); incGeneration(); // Force freeze const auto &store = _store; - const auto saver = _store.getSaver(); - saver.foreach_key([&store,this](EntryRef ref) + const auto enumerator = _store.getEnumerator(); + enumerator.foreach_key([&store,this](EntryRef ref) { const Reference &entry = store.get(ref); _referenceMappings.clearMapping(entry); }); @@ -478,6 +478,6 @@ using Reference = attribute::Reference; template class UniqueStoreAllocator<Reference, EntryRefT<22>>; template class UniqueStore<Reference, EntryRefT<22>>; template class UniqueStoreBuilder<UniqueStoreAllocator<Reference, EntryRefT<22>>>; -template class UniqueStoreSaver<Reference, EntryRefT<22>>; +template class UniqueStoreEnumerator<EntryRefT<22>>; } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp index cb8bb6661d6..e0a6ea193e0 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp @@ -21,7 +21,7 @@ ReferenceAttributeSaver(GenerationHandler::Guard &&guard, : AttributeSaver(std::move(guard), header), _indices(std::move(indices)), _store(store), - _saver(store.getSaver()) + _enumerator(store.getEnumerator()) { } @@ -49,13 +49,13 @@ public: } }; -template <class Store, class Saver> +template <class Store, class Enumerator> void -writeUdat(IAttributeSaveTarget &saveTarget, const Store &store, const Saver &saver) +writeUdat(IAttributeSaveTarget &saveTarget, const Store &store, const Enumerator &enumerator) { std::unique_ptr<BufferWriter> udatWriter(saveTarget.udatWriter().allocBufferWriter()); - saver.foreach_key(ValueWriter<Store>(store, *udatWriter)); + enumerator.foreach_key(ValueWriter<Store>(store, *udatWriter)); udatWriter->flush(); } @@ -64,13 +64,13 @@ writeUdat(IAttributeSaveTarget &saveTarget, const Store &store, const Saver &sav bool ReferenceAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) { - writeUdat(saveTarget, _store, _saver); + writeUdat(saveTarget, _store, _enumerator); std::unique_ptr<search::BufferWriter> datWriter(saveTarget.datWriter(). allocBufferWriter()); - _saver.enumerateValues(); + _enumerator.enumerateValues(); for (const auto &ref : _indices) { - uint32_t enumValue = _saver.mapEntryRefToEnumValue(ref); + uint32_t enumValue = _enumerator.mapEntryRefToEnumValue(ref); datWriter->write(&enumValue, sizeof(uint32_t)); } datWriter->flush(); diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h index 42a26911669..4847663e6e6 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h @@ -5,7 +5,7 @@ #include "attributesaver.h" #include <vespa/document/base/globalid.h> #include <vespa/vespalib/datastore/unique_store.h> -#include <vespa/vespalib/datastore/unique_store_saver.h> +#include <vespa/vespalib/datastore/unique_store_enumerator.h> #include <vespa/vespalib/util/rcuvector.h> #include "reference_attribute.h" #include "reference.h" @@ -32,10 +32,10 @@ private: using GlobalId = document::GlobalId; using IndicesCopyVector = ReferenceAttribute::IndicesCopyVector; using Store = ReferenceAttribute::ReferenceStore; - using Saver = Store::Saver; + using Enumerator = Store::Enumerator; IndicesCopyVector _indices; const Store &_store; - Saver _saver; + Enumerator _enumerator; virtual bool onSave(IAttributeSaveTarget &saveTarget) override; public: diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index c6db5783eba..bde7e3b9f0a 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -123,7 +123,7 @@ struct TestBase : public ::testing::Test { } size_t entrySize() const { return sizeof(ValueType); } auto getBuilder(uint32_t uniqueValuesHint) { return store.getBuilder(uniqueValuesHint); } - auto getSaver() { return store.getSaver(); } + auto getEnumerator() { return store.getEnumerator(); } size_t get_reserved(EntryRef ref) { return store.bufferState(ref).getTypeHandler()->getReservedElements(getBufferId(ref)); } @@ -270,24 +270,24 @@ TYPED_TEST(TestBase, store_can_be_instantiated_with_builder) EXPECT_EQ(val1Ref.ref(), this->add(this->values[1]).ref()); } -TYPED_TEST(TestBase, store_can_be_saved) +TYPED_TEST(TestBase, store_can_be_enumerated) { EntryRef val0Ref = this->add(this->values[0]); EntryRef val1Ref = this->add(this->values[1]); this->remove(this->add(this->values[2])); this->trimHoldLists(); - auto saver = this->getSaver(); + auto enumerator = this->getEnumerator(); std::vector<uint32_t> refs; - saver.foreach_key([&](EntryRef ref) { refs.push_back(ref.ref()); }); + enumerator.foreach_key([&](EntryRef ref) { refs.push_back(ref.ref()); }); std::vector<uint32_t> expRefs; expRefs.push_back(val0Ref.ref()); expRefs.push_back(val1Ref.ref()); EXPECT_EQ(expRefs, refs); - saver.enumerateValues(); - uint32_t invalidEnum = saver.mapEntryRefToEnumValue(EntryRef()); - uint32_t enumValue1 = saver.mapEntryRefToEnumValue(val0Ref); - uint32_t enumValue2 = saver.mapEntryRefToEnumValue(val1Ref); + enumerator.enumerateValues(); + uint32_t invalidEnum = enumerator.mapEntryRefToEnumValue(EntryRef()); + uint32_t enumValue1 = enumerator.mapEntryRefToEnumValue(val0Ref); + uint32_t enumValue2 = enumerator.mapEntryRefToEnumValue(val1Ref); EXPECT_EQ(0u, invalidEnum); EXPECT_EQ(1u, enumValue1); EXPECT_EQ(2u, enumValue2); @@ -318,11 +318,11 @@ TEST_F(DoubleTest, nan_is_handled) EXPECT_FALSE(std::signbit(store.get(refs[2]))); EXPECT_TRUE(std::isinf(store.get(refs[3]))); EXPECT_TRUE(std::signbit(store.get(refs[3]))); - auto saver = getSaver(); - saver.enumerateValues(); + auto enumerator = getEnumerator(); + enumerator.enumerateValues(); std::vector<uint32_t> enumerated; for (auto &ref : refs) { - enumerated.push_back(saver.mapEntryRefToEnumValue(ref)); + enumerated.push_back(enumerator.mapEntryRefToEnumValue(ref)); } std::vector<uint32_t> exp_enumerated = { 0, 1, 4, 2, 3, 1, 4, 2 }; EXPECT_EQ(exp_enumerated, enumerated); diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.hpp b/vespalib/src/vespa/vespalib/datastore/datastore.hpp index 40a00c68774..0e4d1bbe652 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.hpp +++ b/vespalib/src/vespa/vespalib/datastore/datastore.hpp @@ -14,7 +14,7 @@ namespace search::datastore { template <typename RefT> DataStoreT<RefT>::DataStoreT() : DataStoreBase(RefType::numBuffers(), - RefType::offsetSize() / RefType::align(1)) + RefType::unscaled_offset_size()) { } @@ -39,8 +39,7 @@ DataStoreT<RefT>::freeElem(EntryRef ref, size_t numElems) } state.incDeadElems(numElems); state.cleanHold(getBuffer(intRef.bufferId()), - (intRef.offset() / RefType::align(1)) * - state.getArraySize(), numElems); + intRef.unscaled_offset() * state.getArraySize(), numElems); } template <typename RefT> diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h index 9b265abafd3..c65e788637f 100644 --- a/vespalib/src/vespa/vespalib/datastore/entryref.h +++ b/vespalib/src/vespa/vespalib/datastore/entryref.h @@ -40,6 +40,10 @@ public: static size_t align(size_t val) { return val; } static size_t pad(size_t val) { (void) val; return 0ul; } static constexpr bool isAlignedType = false; + // TODO: Remove following temporary methods when removing + // AlignedEntryRefT + size_t unscaled_offset() const { return offset(); } + static size_t unscaled_offset_size() { return offsetSize(); } }; /** diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h index a045da6ca1f..bbf5f9c90a4 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h @@ -19,8 +19,8 @@ namespace search::datastore { template <typename Allocator> class UniqueStoreBuilder; -template <typename EntryT, typename RefT> -class UniqueStoreSaver; +template <typename RefT> +class UniqueStoreEnumerator; /** * Datastore for unique values of type EntryT that is accessed via a @@ -33,7 +33,7 @@ public: using DataStoreType = DataStoreT<RefT>; using EntryType = EntryT; using RefType = RefT; - using Saver = UniqueStoreSaver<EntryT, RefT>; + using Enumerator = UniqueStoreEnumerator<RefT>; using Builder = UniqueStoreBuilder<Allocator>; using EntryConstRefType = typename Allocator::EntryConstRefType; private: @@ -61,7 +61,7 @@ public: uint32_t getNumUniques() const; Builder getBuilder(uint32_t uniqueValuesHint); - Saver getSaver() const; + Enumerator getEnumerator() const; // Should only be used for unit testing const BufferState &bufferState(EntryRef ref) const; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index ae7f82a6e52..1ebdd65d87a 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -8,7 +8,7 @@ #include "unique_store_allocator.hpp" #include "unique_store_builder.hpp" #include "unique_store_dictionary.hpp" -#include "unique_store_saver.hpp" +#include "unique_store_enumerator.hpp" #include <vespa/vespalib/util/bufferwriter.h> #include <atomic> #include <algorithm> @@ -217,10 +217,10 @@ UniqueStore<EntryT, RefT, Compare, Allocator>::getBuilder(uint32_t uniqueValuesH } template <typename EntryT, typename RefT, typename Compare, typename Allocator> -typename UniqueStore<EntryT, RefT, Compare, Allocator>::Saver -UniqueStore<EntryT, RefT, Compare, Allocator>::getSaver() const +typename UniqueStore<EntryT, RefT, Compare, Allocator>::Enumerator +UniqueStore<EntryT, RefT, Compare, Allocator>::getEnumerator() const { - return Saver(*_dict, _store); + return Enumerator(*_dict, _store); } template <typename EntryT, typename RefT, typename Compare, typename Allocator> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_saver.h b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h index d4543cc7d43..94c217bf836 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_saver.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h @@ -2,19 +2,18 @@ #pragma once -#include "unique_store.h" +#include "unique_store_dictionary_base.h" namespace search::datastore { /** - * Saver for related UniqueStore class. + * Enumerator for related UniqueStore class. * * Contains utility methods for traversing all unique values (as * EntryRef value) and mapping from EntryRef value to enum value. */ -template <typename EntryT, typename RefT> -class UniqueStoreSaver { - using EntryType = EntryT; +template <typename RefT> +class UniqueStoreEnumerator { using RefType = RefT; const UniqueStoreDictionaryBase &_dict; @@ -23,8 +22,8 @@ class UniqueStoreSaver { std::vector<std::vector<uint32_t>> _enumValues; uint32_t _next_enum_val; public: - UniqueStoreSaver(const UniqueStoreDictionaryBase &dict, const DataStoreBase &store); - ~UniqueStoreSaver(); + UniqueStoreEnumerator(const UniqueStoreDictionaryBase &dict, const DataStoreBase &store); + ~UniqueStoreEnumerator(); void enumerateValue(EntryRef ref); void enumerateValues(); @@ -38,8 +37,8 @@ public: uint32_t mapEntryRefToEnumValue(EntryRef ref) const { if (ref.valid()) { RefType iRef(ref); - assert(iRef.offset() < _enumValues[iRef.bufferId()].size()); - uint32_t enumValue = _enumValues[iRef.bufferId()][iRef.offset()]; + assert(iRef.unscaled_offset() < _enumValues[iRef.bufferId()].size()); + uint32_t enumValue = _enumValues[iRef.bufferId()][iRef.unscaled_offset()]; assert(enumValue != 0); return enumValue; } else { diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_saver.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp index c29477525c0..b364c2a2ba3 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_saver.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp @@ -2,12 +2,12 @@ #pragma once -#include "unique_store_saver.h" +#include "unique_store_enumerator.h" namespace search::datastore { -template <typename EntryT, typename RefT> -UniqueStoreSaver<EntryT, RefT>::UniqueStoreSaver(const UniqueStoreDictionaryBase &dict, const DataStoreBase &store) +template <typename RefT> +UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const UniqueStoreDictionaryBase &dict, const DataStoreBase &store) : _dict(dict), _root(_dict.get_frozen_root()), _store(store), @@ -15,33 +15,33 @@ UniqueStoreSaver<EntryT, RefT>::UniqueStoreSaver(const UniqueStoreDictionaryBase { } -template <typename EntryT, typename RefT> -UniqueStoreSaver<EntryT, RefT>::~UniqueStoreSaver() +template <typename RefT> +UniqueStoreEnumerator<RefT>::~UniqueStoreEnumerator() { } -template <typename EntryT, typename RefT> +template <typename RefT> void -UniqueStoreSaver<EntryT, RefT>::enumerateValue(EntryRef ref) +UniqueStoreEnumerator<RefT>::enumerateValue(EntryRef ref) { RefType iRef(ref); assert(iRef.valid()); - assert(iRef.offset() < _enumValues[iRef.bufferId()].size()); - uint32_t &enumVal = _enumValues[iRef.bufferId()][iRef.offset()]; + assert(iRef.unscaled_offset() < _enumValues[iRef.bufferId()].size()); + uint32_t &enumVal = _enumValues[iRef.bufferId()][iRef.unscaled_offset()]; assert(enumVal == 0u); enumVal = _next_enum_val; ++_next_enum_val; } -template <typename EntryT, typename RefT> +template <typename RefT> void -UniqueStoreSaver<EntryT, RefT>::enumerateValues() +UniqueStoreEnumerator<RefT>::enumerateValues() { _enumValues.resize(RefType::numBuffers()); for (uint32_t bufferId = 0; bufferId < RefType::numBuffers(); ++bufferId) { const BufferState &state = _store.getBufferState(bufferId); if (state.isActive()) { - _enumValues[bufferId].resize(state.size()); + _enumValues[bufferId].resize(state.size() / state.getArraySize()); } } _next_enum_val = 1; |