diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-10-28 08:52:57 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2020-10-28 08:52:57 +0000 |
commit | 68ca16661cb5a35818f098f325b317a542d55a16 (patch) | |
tree | a4bb65b33d171cb5ed4dd89569ce23a361252d62 /storage/src | |
parent | 4d64cbe4e531cb7be1061f1f54809d1d0a1b0061 (diff) |
Remove legacy Judy array-backed bucket DB implementation
Diffstat (limited to 'storage/src')
16 files changed, 7 insertions, 2232 deletions
diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 2acaea1940d..81c65ec9ad3 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -5,8 +5,6 @@ vespa_add_executable(storage_bucketdb_gtest_runner_app TEST bucketinfotest.cpp bucketmanagertest.cpp gtest_runner.cpp - judyarraytest.cpp - judymultimaptest.cpp lockablemaptest.cpp DEPENDS storage diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp deleted file mode 100644 index 72967e8b4c0..00000000000 --- a/storage/src/tests/bucketdb/judyarraytest.cpp +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/bucketdb/judyarray.h> -#include <boost/random.hpp> -#include <vespa/vespalib/gtest/gtest.h> -#include <gmock/gmock.h> -#include <map> -#include <vector> - -using namespace ::testing; - -namespace storage { - -namespace { - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> - getJudyArrayContents(const JudyArray& array) { - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> vals; - for (JudyArray::const_iterator it = array.begin(); - it != array.end(); ++it) - { - vals.push_back(std::make_pair(it.key(), it.value())); - } - return vals; - } -} - -TEST(JudyArrayTest, iterating) { - JudyArray array; - // Test that things are sane for empty document - ASSERT_EQ(array.begin(), array.end()); - // Add some values - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values({ - {3, 2}, {5, 12}, {15, 8}, {13, 10}, {7, 6}, {9, 4} - }); - for (uint32_t i=0; i<values.size(); ++i) { - array.insert(values[i].first, values[i].second); - } - // Create expected result - std::sort(values.begin(), values.end()); - // Test that we can iterate through const iterator - auto foundVals = getJudyArrayContents(array); - ASSERT_EQ(values, foundVals); - - { // Test that we can alter through non-const iterator - JudyArray::iterator it = array.begin(); - ++it; - ++it; - it.setValue(20); - ASSERT_EQ((JudyArray::key_type) 7, it.key()); - ASSERT_EQ((JudyArray::data_type) 20, array[7]); - it.remove(); - ASSERT_EQ((JudyArray::size_type) 5, getJudyArrayContents(array).size()); - ASSERT_EQ(array.end(), array.find(7)); - values.erase(values.begin() + 2); - ASSERT_EQ(values, getJudyArrayContents(array)); - // And that we can continue iterating after removing. - ++it; - ASSERT_EQ((JudyArray::key_type) 9, it.key()); - ASSERT_EQ((JudyArray::data_type) 4, array[9]); - } - { // Test printing of iterators - JudyArray::ConstIterator cit = array.begin(); - EXPECT_THAT(cit.toString(), MatchesRegex("^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$")); - JudyArray::Iterator it = array.end(); - EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: (0x)?0\\)$")); - } -} - -TEST(JudyArrayTest, dual_array_functions) { - JudyArray array1; - JudyArray array2; - // Add values to array1 - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values1({ - {3, 2}, {5, 12}, {15, 8}, {13, 10}, {7, 6}, {9, 4} - }); - for (uint32_t i=0; i<values1.size(); ++i) { - array1.insert(values1[i].first, values1[i].second); - } - // Add values to array2 - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values2({ - {4, 5}, {9, 40} - }); - for (uint32_t i=0; i<values2.size(); ++i) { - array2.insert(values2[i].first, values2[i].second); - } - // Create expected result - std::sort(values1.begin(), values1.end()); - std::sort(values2.begin(), values2.end()); - - EXPECT_EQ(values1, getJudyArrayContents(array1)); - EXPECT_EQ(values2, getJudyArrayContents(array2)); - EXPECT_LT(array2, array1); - EXPECT_NE(array1, array2); - array1.swap(array2); - EXPECT_EQ(values1, getJudyArrayContents(array2)); - EXPECT_EQ(values2, getJudyArrayContents(array1)); - EXPECT_LT(array1, array2); - EXPECT_NE(array1, array2); - - // Test some operators - JudyArray array3; - for (uint32_t i=0; i<values1.size(); ++i) { - array3.insert(values1[i].first, values1[i].second); - } - EXPECT_NE(array1, array3); - EXPECT_EQ(array2, array3); - EXPECT_FALSE(array2 < array3); -} - -TEST(JudyArrayTest, size) { - JudyArray array; - EXPECT_EQ(array.begin(), array.end()); - EXPECT_TRUE(array.empty()); - EXPECT_EQ((JudyArray::size_type) 0, array.size()); - EXPECT_EQ((JudyArray::size_type) 0, array.getMemoryUsage()); - - // Test each method one can insert stuff into array - array.insert(4, 3); - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - array.insert(4, 7); - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - EXPECT_EQ((JudyArray::size_type) 24, array.getMemoryUsage()); - - array[6] = 8; - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - array[6] = 10; - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - EXPECT_EQ((JudyArray::size_type) 40, array.getMemoryUsage()); - - bool preExisted; - array.find(8, true, preExisted); - EXPECT_EQ(false, preExisted); - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - array.find(8, true, preExisted); - EXPECT_EQ(true, preExisted); - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - EXPECT_EQ((JudyArray::size_type) 3, array.size()); - EXPECT_EQ((JudyArray::size_type) 56, array.getMemoryUsage()); - - // Test each method one can remove stuff in array with - array.erase(8); - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - array.erase(8); - EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); - EXPECT_EQ((JudyArray::size_type) 2, array.size()); - EXPECT_EQ((JudyArray::size_type) 40, array.getMemoryUsage()); -} - -TEST(JudyArrayTest, stress) { - // Do a lot of random stuff to both judy array and std::map. Ensure equal - // behaviour - - JudyArray judyArray; - typedef std::map<JudyArray::key_type, JudyArray::data_type> StdMap; - StdMap stdMap; - - boost::rand48 rnd(55); - - for (uint32_t checkpoint=0; checkpoint<50; ++checkpoint) { - for (uint32_t opnr=0; opnr<500; ++opnr) { - int optype = rnd() % 100; - if (optype < 30) { // Insert - JudyArray::key_type key(rnd() % 500); - JudyArray::key_type value(rnd()); - judyArray.insert(key, value); - stdMap[key] = value; - } else if (optype < 50) { // operator[] - JudyArray::key_type key(rnd() % 500); - JudyArray::key_type value(rnd()); - judyArray[key] = value; - stdMap[key] = value; - } else if (optype < 70) { // erase() - JudyArray::key_type key(rnd() % 500); - EXPECT_EQ(stdMap.erase(key), judyArray.erase(key)); - } else if (optype < 75) { // size() - EXPECT_EQ(stdMap.size(), judyArray.size()); - } else if (optype < 78) { // empty() - EXPECT_EQ(stdMap.empty(), judyArray.empty()); - } else { // find() - JudyArray::key_type key(rnd() % 500); - auto it = judyArray.find(key); - auto it2 = stdMap.find(key); - EXPECT_EQ(it2 == stdMap.end(), it == judyArray.end()); - if (it != judyArray.end()) { - EXPECT_EQ(it.key(), it2->first); - EXPECT_EQ(it.value(), it2->second); - } - } - } - // Ensure judy array contents is equal to std::map's at this point - StdMap tmpMap; - for (JudyArray::const_iterator it = judyArray.begin(); - it != judyArray.end(); ++it) - { - tmpMap[it.key()] = it.value(); - } - EXPECT_EQ(stdMap, tmpMap); - } -} - -} // storage diff --git a/storage/src/tests/bucketdb/judymultimaptest.cpp b/storage/src/tests/bucketdb/judymultimaptest.cpp deleted file mode 100644 index 254dbb78b18..00000000000 --- a/storage/src/tests/bucketdb/judymultimaptest.cpp +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/bucketdb/judymultimap.h> -#include <vespa/storage/bucketdb/judymultimap.hpp> -#include <vespa/vespalib/gtest/gtest.h> -#include <map> -#include <ostream> -#include <vector> - -using namespace ::testing; - -namespace storage { - -namespace { - struct B; - struct C; - - struct A { - int _val1; - int _val2; - int _val3; - - A() = default; - A(const B &); - A(const C &); - A(int val1, int val2, int val3) - : _val1(val1), _val2(val2), _val3(val3) {} - - static bool mayContain(const A&) { return true; } - - bool operator==(const A& a) const { - return (_val1 == a._val1 && _val2 == a._val2 && _val3 == a._val3); - } - }; - - struct B { - int _val1; - int _val2; - - B() = default; - B(const A& a) : _val1(a._val1), _val2(a._val2) {} - B(int val1, int val2) : _val1(val1), _val2(val2) {} - - static bool mayContain(const A& a) { return (a._val3 == 0); } - }; - - struct C { - int _val1; - - C() = default; - C(const A& a) : _val1(a._val1) {} - C(int val1) : _val1(val1) {} - - static bool mayContain(const A& a) { return (a._val2 == 0 && a._val3 == 0); } - }; - - A::A(const B& b) : _val1(b._val1), _val2(b._val2), _val3(0) {} - A::A(const C& c) : _val1(c._val1), _val2(0), _val3(0) {} - - std::ostream& operator<<(std::ostream& out, const A& a) { - return out << "A(" << a._val1 << ", " << a._val2 << ", " << a._val3 << ")"; - } - std::ostream& operator<<(std::ostream& out, const B& b) { - return out << "B(" << b._val1 << ", " << b._val2 << ")"; - } - std::ostream& operator<<(std::ostream& out, const C& c) { - return out << "C(" << c._val1 << ")"; - } -} - -TEST(JudyMultiMapTest, simple_usage) { - typedef JudyMultiMap<C, B, A> MultiMap; - MultiMap multiMap; - // Do some insertions - bool preExisted; - EXPECT_TRUE(multiMap.empty()); - multiMap.insert(16, A(1, 2, 3), preExisted); - EXPECT_EQ(false, preExisted); - multiMap.insert(11, A(4, 6, 0), preExisted); - EXPECT_EQ(false, preExisted); - multiMap.insert(14, A(42, 0, 0), preExisted); - EXPECT_EQ(false, preExisted); - EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()) << multiMap.toString(); - - multiMap.insert(11, A(4, 7, 0), preExisted); - EXPECT_EQ(true, preExisted); - EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()); - EXPECT_FALSE(multiMap.empty()); - - // Access some elements - EXPECT_EQ(A(4, 7, 0), multiMap[11]); - EXPECT_EQ(A(1, 2, 3), multiMap[16]); - EXPECT_EQ(A(42,0, 0), multiMap[14]); - - // Do removes - EXPECT_EQ(multiMap.erase(12), 0); - EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()); - - EXPECT_EQ(multiMap.erase(14), 1); - EXPECT_EQ((MultiMap::size_type) 2, multiMap.size()); - - EXPECT_EQ(multiMap.erase(11), 1); - EXPECT_EQ(multiMap.erase(16), 1); - EXPECT_EQ((MultiMap::size_type) 0, multiMap.size()); - EXPECT_TRUE(multiMap.empty()); -} - -TEST(JudyMultiMapTest, iterator) { - typedef JudyMultiMap<C, B, A> MultiMap; - MultiMap multiMap; - bool preExisted; - // Do some insertions - multiMap.insert(16, A(1, 2, 3), preExisted); - multiMap.insert(11, A(4, 6, 0), preExisted); - multiMap.insert(14, A(42, 0, 0), preExisted); - - MultiMap::Iterator iter = multiMap.begin(); - EXPECT_EQ((uint64_t)11, (uint64_t)iter.key()); - EXPECT_EQ(A(4, 6, 0), iter.value()); - ++iter; - EXPECT_EQ((uint64_t)14, (uint64_t)iter.key()); - EXPECT_EQ(A(42, 0, 0), iter.value()); - ++iter; - EXPECT_EQ((uint64_t)16, (uint64_t)iter.key()); - EXPECT_EQ(A(1, 2, 3), iter.value()); - --iter; - EXPECT_EQ((uint64_t)14, (uint64_t)iter.key()); - EXPECT_EQ(A(42, 0, 0), iter.value()); - ++iter; - EXPECT_EQ((uint64_t)16, (uint64_t)iter.key()); - EXPECT_EQ(A(1, 2, 3), iter.value()); - --iter; - --iter; - EXPECT_EQ((uint64_t)11,(uint64_t) iter.key()); - EXPECT_EQ(A(4, 6, 0), iter.value()); - ++iter; - ++iter; - ++iter; - EXPECT_EQ(multiMap.end(), iter); - --iter; - EXPECT_EQ((uint64_t)16, (uint64_t)iter.key()); - EXPECT_EQ(A(1, 2, 3), iter.value()); - --iter; - EXPECT_EQ((uint64_t)14, (uint64_t)iter.key()); - EXPECT_EQ(A(42, 0, 0), iter.value()); - --iter; - EXPECT_EQ((uint64_t)11,(uint64_t) iter.key()); - EXPECT_EQ(A(4, 6, 0), iter.value()); -} - -} // storage - diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 1ea10003de1..f5e68dd3de1 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/util/document_runnable.h> -#include <vespa/storage/bucketdb/judymultimap.h> -#include <vespa/storage/bucketdb/judymultimap.hpp> -#include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/storage/bucketdb/btree_lockable_map.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> @@ -58,7 +55,8 @@ struct LockableMapTest : ::testing::Test { using Map = MapT; }; -using MapTypes = ::testing::Types<LockableMap<JudyMultiMap<A>>, bucketdb::BTreeLockableMap<A>>; +// TODO add striped B-tree DB to this type set once ready +using MapTypes = ::testing::Types<bucketdb::BTreeLockableMap<A>>; VESPA_GTEST_TYPED_TEST_SUITE(LockableMapTest, MapTypes); // Disable warnings emitted by gtest generated files when using typed tests diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 5f491d80697..23974cf2522 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -9,8 +9,6 @@ vespa_add_library(storage_bucketdb OBJECT bucketmanager.cpp bucketmanagermetrics.cpp generic_btree_bucket_database.cpp - judyarray.cpp - lockablemap.cpp storbucketdb.cpp DEPENDS ) diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 392c2da871a..7b934af4bdd 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -2,7 +2,6 @@ #include "bucketmanager.h" #include "minimumusedbitstracker.h" -#include "lockablemap.hpp" #include <iomanip> #include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/nodestateupdater.h> @@ -22,6 +21,7 @@ #include <vespa/config/config.h> #include <vespa/config/helper/configgetter.hpp> #include <chrono> +#include <thread> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".storage.bucketdb.manager"); diff --git a/storage/src/vespa/storage/bucketdb/judyarray.cpp b/storage/src/vespa/storage/bucketdb/judyarray.cpp deleted file mode 100644 index cd348300d28..00000000000 --- a/storage/src/vespa/storage/bucketdb/judyarray.cpp +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "judyarray.h" -#include <vespa/vespalib/util/exceptions.h> -#include <iostream> -#include <sstream> - -namespace storage { - -JudyArray::~JudyArray() -{ - clear(); -} - -JudyArray::iterator -JudyArray::find(key_type key, bool insertIfNonExisting, bool& preExisted) -{ - Iterator iter(*this, key); - if (insertIfNonExisting && (iter.end() || iter.key() != key)) { - preExisted = false; - insert(key, 0); - iter = Iterator(*this, key); - assert(iter.key() == key); - } else if (iter.key() != key) { - preExisted = false; - iter = Iterator(*this); - } else { - preExisted = true; - } - return iter; -} - -void -JudyArray::Iterator::setValue(data_type val) -{ - if (_data == 0) { - throw vespalib::IllegalArgumentException( - "Cannot set value of end() iterator", VESPA_STRLOC); - } - *_data = val; -} - -void -JudyArray::Iterator::remove() -{ - if (_data == 0) { - throw vespalib::IllegalArgumentException( - "Cannot erase end() iterator", VESPA_STRLOC); - } - _parent->erase(_key); -} - -bool -JudyArray::operator==(const JudyArray& array) const -{ - if (size() != array.size()) return false; - for (JudyArray::const_iterator it1 = begin(), it2 = array.begin(); - it1 != end(); ++it1, ++it2) - { - if (*it1 != *it2) return false; - } - return true; -} - -bool -JudyArray::operator<(const JudyArray& array) const -{ - if (size() != array.size()) return (size() < array.size()); - for (JudyArray::const_iterator it1 = begin(), it2 = array.begin(); - it1 != end(); ++it1, ++it2) - { - if (*it1 != *it2) return (*it1 < *it2); - } - return false; -} - -JudyArray::size_type -JudyArray::erase(key_type key) -{ - JError_t err; - size_type result = JudyLDel(&_judyArray, key, &err); - if (result == 0 || result == 1) { - return result; - } - std::ostringstream ost; - ost << "Judy error in erase(" << std::hex << key << "): " << err.je_Errno; - std::cerr << ost.str() << "\n"; - assert(false); - return 0; -} - - -JudyArray::size_type -JudyArray::size() const -{ - key_type lastIndex = 0; - --lastIndex; // Get last index in size independent way - return JudyLCount(_judyArray, 0, lastIndex, PJE0); -} - -void -JudyArray::swap(JudyArray& other) -{ - void* judyArray = _judyArray; // Save our variables - _judyArray = other._judyArray; // Assign others to ours - other._judyArray = judyArray; // Assign temporary to other -} - -void -JudyArray::print(std::ostream& out, bool, const std::string& indent) const -{ - out << "JudyArray("; - for (const_iterator i = begin(); i != end(); ++i) { - out << "\n" << indent << " Key: " << i.key() - << ", Value: " << i.value(); - } - out << "\n" << indent << ")"; -} - -JudyArray::ConstIterator::ConstIterator(const JudyArray& arr) - : _key(0), _data(0), _parent(const_cast<JudyArray*>(&arr)) {} - -JudyArray::ConstIterator::ConstIterator(const JudyArray& arr, key_type mykey) - : _key(mykey), _data(0), _parent(const_cast<JudyArray*>(&arr)) -{ - _data = reinterpret_cast<data_type*>( - JudyLFirst(_parent->_judyArray, &_key, PJE0)); -} - -void -JudyArray::ConstIterator::print(std::ostream& out, bool, const std::string&) const -{ - if (dynamic_cast<const Iterator*>(this) == 0) { - out << "Const"; - } - out << "Iterator(Key: " << _key << ", Valp: " << _data; - if (_data) out << ", Val: " << *_data; - out << ")"; -} - -JudyArray::Iterator::Iterator(JudyArray& arr) - : ConstIterator(arr) {} - -JudyArray::Iterator::Iterator(JudyArray& arr, key_type mykey) - : ConstIterator(arr, mykey) {} - -} // storage diff --git a/storage/src/vespa/storage/bucketdb/judyarray.h b/storage/src/vespa/storage/bucketdb/judyarray.h deleted file mode 100644 index 71db95d42c0..00000000000 --- a/storage/src/vespa/storage/bucketdb/judyarray.h +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class JudyArray - * - * Implements a pair associative container on top of a judy array. - * - * NB: All iterators are invalidated after writing to judy array. - * - * NB: Using JudyArray's insert, one can only detect if the element already - * existed, if the element didn't have the value 0. Since we don't want to - * say that values cannot be 0, size is not counted outside of judy array, but - * rather counts elements in the judy array when asked. - * - * @author Haakon Humberset - */ - -#pragma once - -#include <vespa/vespalib/util/printable.h> -#include <cinttypes> -#include <Judy.h> -#include <cassert> - -namespace storage { - -class JudyArray : public vespalib::Printable -{ - JudyArray(const JudyArray&); // Deny copying - JudyArray& operator=(const JudyArray&); - -public: - class Iterator; - class ConstIterator; - - typedef Iterator iterator; - typedef ConstIterator const_iterator; - typedef unsigned long key_type; - typedef unsigned long data_type; - typedef std::pair<const key_type, data_type> value_type; - typedef size_t size_type; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef value_type* pointer; - typedef int difference_type; - - JudyArray() : _judyArray(NULL) {} - virtual ~JudyArray(); - - bool operator==(const JudyArray& array) const; - bool operator!=(const JudyArray& array) const { - return ! (*this == array); - } - bool operator<(const JudyArray& array) const; - - /** Warning: Size may be a O(n) function (Unknown implementation in judy) */ - size_type size() const; - bool empty() const { return (begin() == end()); } - - iterator begin() { return Iterator(*this, 0); } - iterator end() { return Iterator(*this); } - const_iterator begin() const { return ConstIterator(*this, 0); } - const_iterator end() const { return ConstIterator(*this); } - - void swap(JudyArray&); - - const_iterator find(key_type key) const; - /** - * Get iterator to value with given key. If non-existing, returns end(), - * unless insert is true, in which case the element will be created. - */ - iterator find(key_type key, bool insert, bool& preExisted); - iterator find(key_type key) { bool b; return find(key, false, b); } - - const_iterator lower_bound(key_type key) const - { return ConstIterator(*this, key); } - iterator lower_bound(key_type key) { return Iterator(*this, key); } - - size_type erase(key_type key); - void erase(iterator& iter) { iter.remove(); } - - void insert(key_type key, data_type val); - void clear(); - - data_type& operator[](key_type key); - size_type getMemoryUsage() const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - class ConstIterator : public vespalib::Printable - { - public: - ConstIterator& operator--(); - ConstIterator& operator++(); - - bool operator==(const ConstIterator &cp) const; - bool operator!=(const ConstIterator &cp) const { - return ! (*this == cp); - } - value_type operator*() const { return value_type(_key, *_data); } - - bool end() const { return (_data == 0); } - key_type key() const { return _key; } - data_type value() const { return *_data; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - protected: - // For creating end() iterator - ConstIterator(const JudyArray&); - // Create iterator pointing to first element >= key. - ConstIterator(const JudyArray&, key_type); - - key_type _key; // Key iterator currently points to - data_type* _data; // Pointer to member pointed to, or 0 if end(). - JudyArray* _parent; - friend class JudyArray; - }; - - class Iterator : public ConstIterator - { - public: - Iterator& operator--() - { return static_cast<Iterator&>(ConstIterator::operator--()); } - - Iterator& operator++() - { return static_cast<Iterator&>(ConstIterator::operator++()); } - - void setValue(data_type val); - void remove(); - - private: - Iterator(JudyArray&); - Iterator(JudyArray&, key_type key); - friend class JudyArray; - }; - -private: - void *_judyArray; - friend class Iterator; - friend class ConstIterator; -}; - -inline JudyArray::const_iterator -JudyArray::find(key_type key) const -{ - ConstIterator iter(*this, key); - if (!iter.end() && iter.key() != key) { - iter = ConstIterator(*this); - } - return iter; -} - -inline void -JudyArray::insert(key_type key, data_type val) -{ - data_type* valp = reinterpret_cast<data_type*>( - JudyLIns(&_judyArray, key, PJE0)); - *valp = val; -} - -inline void -JudyArray::clear() -{ - JudyLFreeArray(&_judyArray, PJE0); -} - -inline JudyArray::data_type& -JudyArray::operator[](key_type key) -{ - data_type* valp = reinterpret_cast<data_type*>( - JudyLGet(_judyArray, key, PJE0)); - if (valp == 0) { - valp = reinterpret_cast<data_type*>(JudyLIns(&_judyArray, key, PJE0)); - *valp = 0; - } - return *valp; -} - -inline JudyArray::size_type -JudyArray::getMemoryUsage() const -{ - return JudyLMemUsed(_judyArray); -} - -inline JudyArray::ConstIterator& -JudyArray::ConstIterator::operator--() // Prefix -{ - if (!_data) { - _data = reinterpret_cast<data_type*>( - JudyLLast(_parent->_judyArray, &_key, PJE0)); - } else { - _data = reinterpret_cast<data_type*>( - JudyLPrev(_parent->_judyArray, &_key, PJE0)); - } - return *this; -} - -inline JudyArray::ConstIterator& -JudyArray::ConstIterator::operator++() // Prefix -{ - _data = reinterpret_cast<data_type*>( - JudyLNext(_parent->_judyArray, &_key, PJE0)); - return *this; -} - -inline bool -JudyArray::ConstIterator::operator==(const JudyArray::ConstIterator &cp) const -{ - return (_data == cp._data); -} - -} // storage diff --git a/storage/src/vespa/storage/bucketdb/judymultimap.h b/storage/src/vespa/storage/bucketdb/judymultimap.h deleted file mode 100644 index b321f6a4dea..00000000000 --- a/storage/src/vespa/storage/bucketdb/judymultimap.h +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class JudyMultiMap - * - * Layer on top of JudyArray, to create a map from the judy array key type, - * to any of a given set of array types. - * - * The value arrays in here all starts with an unused object at index 0. - * This is because 0 is used as unset value in judyarray, such that we can - * easily detect if we replace or insert new entry. - * - * NB: The order of the template parameters type must be ordered such that - * the types can include less and less. - * - * NB: All iterators are invalidated after writing to judy map. - * - * NB: Using JudyArray's insert, one can only detect if the element already - * existed, if the element didn't have the value 0. Since we don't want to - * say that values cannot be 0, size is not counted outside of judy array, but - * rather counts elements in the judy array when asked. - * - * @author Haakon Humberset< - */ - - -#pragma once - -#include <vespa/storage/bucketdb/judyarray.h> -#include <vespa/vespalib/util/array.h> -#include <vector> - -namespace storage { - -template<class Type0, - class Type1 = Type0, - class Type2 = Type1, - class Type3 = Type2 > -class JudyMultiMap : public vespalib::Printable { -public: - JudyMultiMap(); - ~JudyMultiMap(); - - class Iterator; - class ConstIterator; - class ValueType; - - typedef Iterator iterator; - typedef ConstIterator const_iterator; - typedef JudyArray::key_type key_type; - typedef Type3 mapped_type; - typedef std::pair<const key_type, mapped_type> value_type; - typedef JudyArray::size_type size_type; - - bool operator==(const JudyMultiMap& array) const; - bool operator<(const JudyMultiMap& array) const; - - /** Warning: Size may be a O(n) function (Unknown implementation in judy) */ - size_type size() const; - bool empty() const { return (begin() == end()); } - - iterator begin() { return Iterator(*this, 0); } - iterator end() { return Iterator(*this); } - const_iterator begin() const { return ConstIterator(*this, 0); } - const_iterator end() const { return ConstIterator(*this); } - - void swap(JudyMultiMap&); - - const_iterator find(key_type key) const; - /** - * Get iterator to value with given key. If non-existing, returns end(), - * unless insert is true, in which case the element will be created. - */ - iterator find(key_type key, bool insert, bool& preExisted); - iterator find(key_type key) { bool b; return find(key, false, b); } - - const_iterator lower_bound(key_type key) const - { return ConstIterator(*this, key); } - iterator lower_bound(key_type key) { return Iterator(*this, key); } - - size_type erase(key_type key); -#if 0 - void erase(iterator& iter) { iter.remove(); } -#endif - - void insert(key_type key, const Type3& val, bool& preExisted) - { - JudyArray::iterator it(_judyArray.find(key, true, preExisted)); - insert(it, val); - } - void clear(); - - const mapped_type operator[](key_type key); - size_type getMemoryUsage() const; - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - class ConstIterator : public vespalib::Printable - { - public: - ConstIterator& operator--() { --_iterator; return *this; } - ConstIterator& operator++() { ++_iterator; return *this; } - - bool operator==(const ConstIterator &cp) const; - bool operator!=(const ConstIterator &cp) const { - return ! (*this == cp); - } - value_type operator*() const; - - bool end() const { return _iterator.end(); } - key_type key() const { return _iterator.key(); } - mapped_type value() const; - - const std::pair<key_type, mapped_type>* operator->() const { - _pair = std::pair<key_type, mapped_type>(_iterator.key(), value()); - return &_pair; - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - protected: - // For creating end() iterator - ConstIterator(const JudyMultiMap&); - // Create iterator pointing to first element >= key. - ConstIterator(const JudyMultiMap&, key_type); - - JudyArray::ConstIterator _iterator; - JudyMultiMap* _parent; - friend class JudyMultiMap; - mutable std::pair<key_type, mapped_type> _pair; - }; - - class Iterator : public ConstIterator - { - public: - Iterator& operator--() - { return static_cast<Iterator&>(ConstIterator::operator--()); } - - Iterator& operator++() - { return static_cast<Iterator&>(ConstIterator::operator++()); } - - void setValue(const Type3& val); - void remove(); - - private: - Iterator(JudyMultiMap&); - Iterator(JudyMultiMap&, key_type key); - friend class JudyMultiMap; - }; - -private: - JudyArray _judyArray; - typedef vespalib::Array<Type0> Type0Vector; - typedef vespalib::Array<Type1> Type1Vector; - typedef vespalib::Array<Type2> Type2Vector; - typedef vespalib::Array<Type3> Type3Vector; - Type0Vector _values0; - Type1Vector _values1; - Type2Vector _values2; - Type3Vector _values3; - std::vector<std::vector<typename Type0Vector::size_type> > _free; - friend class Iterator; - friend class ConstIterator; - - static int getType(JudyArray::data_type index) { - return index >> (8 * sizeof(JudyArray::data_type) - 2); - } - static JudyArray::data_type getIndex(JudyArray::data_type index) { - return ((index << 2) >> 2); - } - static JudyArray::data_type getValue(JudyArray::data_type type, - JudyArray::data_type index) - { - return (type << (8 * sizeof(JudyArray::data_type) - 2) | index); - } - void insert(JudyArray::iterator& it, const Type3& val); -}; - -} // storage - diff --git a/storage/src/vespa/storage/bucketdb/judymultimap.hpp b/storage/src/vespa/storage/bucketdb/judymultimap.hpp deleted file mode 100644 index 0d80bcfb28e..00000000000 --- a/storage/src/vespa/storage/bucketdb/judymultimap.hpp +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "judymultimap.h" -#include <vespa/vespalib/util/hdr_abort.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/array.hpp> -#include <set> -#include <ostream> - -namespace storage { - -template<class T0, class T1, class T2, class T3> -JudyMultiMap<T0, T1, T2, T3>::JudyMultiMap() - : _values0(1), _values1(1), _values2(1), _values3(1), _free(4) -{ } - -template<class T0, class T1, class T2, class T3> -JudyMultiMap<T0, T1, T2, T3>::~JudyMultiMap() { } - -template<class T0, class T1, class T2, class T3> -bool -JudyMultiMap<T0, T1, T2, T3>:: -operator==(const JudyMultiMap<T0, T1, T2, T3>& map) const -{ - if (size() != map.size()) return false; - for (typename JudyMultiMap<T0, T1, T2, T3>::const_iterator - it1 = begin(), it2 = map.begin(); it1 != end(); ++it1, ++it2) - { - assert(it2 != end()); - if (*it1 != *it2) return false; - } - return true; -} - -template<class T0, class T1, class T2, class T3> -bool -JudyMultiMap<T0, T1, T2, T3>:: -operator<(const JudyMultiMap<T0, T1, T2, T3>& map) const -{ - if (size() != map.size()) return (size() < map.size()); - for (typename JudyMultiMap<T0, T1, T2, T3>::const_iterator - it1 = begin(), it2 = map.begin(); it1 != end(); ++it1, ++it2) - { - if (it1.key() != it2.key()) return (it1.key() < it2.key()); - if (it1.value() != it2.value()) return (it1.value() < it2.value()); - } - return false; -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::size_type -JudyMultiMap<T0, T1, T2, T3>::size() const -{ - // First elements in all vectors is bogus, because we use value 0 - // to mean unset in judyarray. (To be able to detect if we overwrite) - return _values0.size() + _values1.size() - + _values2.size() + _values3.size() - 4 - - _free[0].size() - _free[1].size() - - _free[2].size() - _free[3].size(); -} - -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>:: -swap(JudyMultiMap<T0, T1, T2, T3>& other) -{ - _judyArray.swap(other._judyArray); - _values0.swap(other._values0); - _values1.swap(other._values1); - _values2.swap(other._values2); - _values3.swap(other._values3); - _free.swap(other._free); -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::const_iterator -JudyMultiMap<T0, T1, T2, T3>::find(key_type key) const -{ - ConstIterator iter(*this, key); - if (!iter.end() && iter.key() != key) { - iter = ConstIterator(*this); - } - return iter; -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::iterator -JudyMultiMap<T0, T1, T2, T3>::find(key_type key, bool insertIfNonExisting, - bool& preExisted) -{ - Iterator iter(*this, key); - if (insertIfNonExisting && (iter.end() || iter.key() != key)) { - insert(key, T3(), preExisted); - iter = Iterator(*this, key); - assert(iter.key() == key); - } else if (iter.key() != key) { - preExisted = false; - iter = Iterator(*this); - } else { - preExisted = true; - } - return iter; -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::size_type -JudyMultiMap<T0, T1, T2, T3>::erase(key_type key) -{ - JudyArray::iterator it = _judyArray.find(key); - if (it == _judyArray.end()) return 0; - _free[getType(it.value())].push_back(getIndex(it.value())); - _judyArray.erase(key); - return 1; -} - -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>::clear() -{ - _judyArray.clear(); - _values0.resize(1); - _values1.resize(1); - _values2.resize(1); - _values3.resize(1); - _free[0].clear(); - _free[1].clear(); - _free[2].clear(); - _free[3].clear(); -} - -template<class T0, class T1, class T2, class T3> -const typename JudyMultiMap<T0, T1, T2, T3>::mapped_type -JudyMultiMap<T0, T1, T2, T3>::operator[](key_type key) -{ - bool preExisted; - JudyArray::iterator it = _judyArray.find(key, true, preExisted); - // If it doesn't already exist, insert - if (it.value() == 0) { - if (_free[0].empty()) { - it.setValue(getValue(0, _values0.size())); - _values0.push_back(T0()); - } else { - it.setValue(getValue(0, _free[0].back())); - _values0[_free[0].back()] = T0(); - _free[0].pop_back(); - } - } - switch (getType(it.value())) { - case 0: return _values0[getIndex(it.value())]; - case 1: return _values1[getIndex(it.value())]; - case 2: return _values2[getIndex(it.value())]; - case 3: return _values3[getIndex(it.value())]; - default: HDR_ABORT("should not be reached"); - } - return T0(); // Avoid warning of no return -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::size_type -JudyMultiMap<T0, T1, T2, T3>::getMemoryUsage() const -{ - return _judyArray.getMemoryUsage() - + sizeof(T0) * _values0.capacity() - + sizeof(T1) * _values1.capacity() - + sizeof(T2) * _values2.capacity() - + sizeof(T3) * _values3.capacity() - + sizeof(typename Type0Vector::size_type) - * (_free[0].capacity() + _free[1].capacity() + - _free[2].capacity() + _free[3].capacity()); -} - -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>:: -print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "JudyMultiMap("; - - if (verbose) { - for (const_iterator i = begin(); i != end(); ++i) { - out << "\n" << indent << " "; - i.print(out, verbose, indent + " "); - } - } - - if (_values0.size() > 1) { - std::set<typename Type0Vector::size_type> free( - _free[0].begin(), _free[0].end()); - assert(free.size() == _free[0].size()); - out << "\n" << indent << " Type0 " << (_values0.size()-1) - << " entries, " << free.size() << " free {"; - - if (verbose) { - for (uint32_t i=1; i<_values0.size(); ++i) { - out << "\n" << indent << " "; - if (free.find(i) != free.end()) { out << "free"; } - else { out << _values0[i]; } - } - } - out << "\n" << indent << " }"; - } - if (_values1.size() > 1) { - std::set<typename Type0Vector::size_type> free( - _free[1].begin(), _free[1].end()); - assert(free.size() == _free[1].size()); - out << "\n" << indent << " Type1 " << (_values1.size()-1) - << " entries, " << free.size() << " free {"; - if (verbose) { - for (uint32_t i=1; i<_values1.size(); ++i) { - out << "\n" << indent << " "; - if (free.find(i) != free.end()) { out << "free"; } - else { out << _values1[i]; } - } - } - out << "\n" << indent << " }"; - } - if (_values2.size() > 1) { - std::set<typename Type0Vector::size_type> free( - _free[2].begin(), _free[2].end()); - assert(free.size() == _free[2].size()); - out << "\n" << indent << " Type2 " << (_values2.size()-1) - << " entries, " << free.size() << " free {"; - if (verbose) { - for (uint32_t i=1; i<_values2.size(); ++i) { - out << "\n" << indent << " "; - if (free.find(i) != free.end()) { out << "free"; } - else { out << _values2[i]; } - } - } - out << "\n" << indent << " }"; - } - - if (_values3.size() > 1) { - std::set<typename Type0Vector::size_type> free( - _free[3].begin(), _free[3].end()); - assert(free.size() == _free[3].size()); - out << "\n" << indent << " Type3 " << (_values3.size()-1) - << " entries, " << free.size() << " free {"; - - if (verbose) { - for (uint32_t i=1; i<_values3.size(); ++i) { - out << "\n" << indent << " "; - if (free.find(i) != free.end()) { out << "free"; } - else { out << _values3[i]; } - } - } - out << "\n" << indent << " }"; - } - if (!empty()) { out << "\n" << indent; } - out << ")"; -} - -template<class T0, class T1, class T2, class T3> -JudyMultiMap<T0, T1, T2, T3>:: -ConstIterator::ConstIterator(const JudyMultiMap<T0, T1, T2, T3>& map) - : _iterator(map._judyArray.end()), - _parent(const_cast<JudyMultiMap<T0, T1, T2, T3>*>(&map)) -{ -} - -template<class T0, class T1, class T2, class T3> -JudyMultiMap<T0, T1, T2, T3>:: -ConstIterator::ConstIterator(const JudyMultiMap<T0, T1, T2, T3>& map, - key_type mykey) - : _iterator(map._judyArray.lower_bound(mykey)), - _parent(const_cast<JudyMultiMap<T0, T1, T2, T3>*>(&map)) -{ -} - -template<class T0, class T1, class T2, class T3> -bool -JudyMultiMap<T0, T1, T2, T3>:: -ConstIterator::operator==(const JudyMultiMap::ConstIterator &cp) const -{ - return (_iterator == cp._iterator); -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::value_type -JudyMultiMap<T0, T1, T2, T3>::ConstIterator::operator*() const -{ - switch (getType(_iterator.value())) { - case 0: return value_type( - _iterator.key(), _parent->_values0[getIndex(_iterator.value())]); - case 1: return value_type( - _iterator.key(), _parent->_values1[getIndex(_iterator.value())]); - case 2: return value_type( - _iterator.key(), _parent->_values2[getIndex(_iterator.value())]); - case 3: return value_type( - _iterator.key(), _parent->_values3[getIndex(_iterator.value())]); - default: - HDR_ABORT("should not be reached"); - } -} - -template<class T0, class T1, class T2, class T3> -typename JudyMultiMap<T0, T1, T2, T3>::mapped_type -JudyMultiMap<T0, T1, T2, T3>::ConstIterator::value() const -{ - switch (getType(_iterator.value())) { - case 0: return _parent->_values0[getIndex(_iterator.value())]; - case 1: return _parent->_values1[getIndex(_iterator.value())]; - case 2: return _parent->_values2[getIndex(_iterator.value())]; - case 3: return _parent->_values3[getIndex(_iterator.value())]; - default: - HDR_ABORT("should not be reached"); - } -} - -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>:: -ConstIterator::print(std::ostream& out, bool, const std::string&) const -{ - if (dynamic_cast<const Iterator*>(this) == 0) { - out << "Const"; - } - out << "Iterator(Key: " << _iterator.key() << ", Value: " << value() << ")"; -} - -template<class T0, class T1, class T2, class T3> -JudyMultiMap<T0, T1, T2, T3>:: -Iterator::Iterator(JudyMultiMap<T0, T1, T2, T3>& map) - : ConstIterator(map) {} - -template<class T0, class T1, class T2, class T3> -JudyMultiMap<T0, T1, T2, T3>:: -Iterator::Iterator(JudyMultiMap<T0, T1, T2, T3>& map, key_type mykey) - : ConstIterator(map, mykey) {} - -#if 0 -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>::Iterator::setValue(const T3& val) -{ - if (this->_iterator.end()) { - throw vespalib::IllegalArgumentException( - "Cannot set value of end() iterator", VESPA_STRLOC); - } - insert(this->iterator, val); -} - -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>::Iterator::remove() -{ - if (this->_iterator.end()) { - throw vespalib::IllegalArgumentException( - "Cannot erase end() iterator", VESPA_STRLOC); - } - int type = getType(this->_iterator.value()); - _free[type].push_back(getIndex(this->_iterator.value())); - this->_iterator.remove(); -} - -#endif - -template<class T0, class T1, class T2, class T3> -void -JudyMultiMap<T0, T1, T2, T3>::insert(JudyArray::iterator& it, const T3& val) -{ - // Find the type we need to save 'val' as - int type; - if (T0::mayContain(val)) { type = 0; } - else if (T1::mayContain(val)) { type = 1; } - else if (T2::mayContain(val)) { type = 2; } - else { type = 3; } - // If already pointing to some value, free that resource. - int oldtype = getType(it.value()); - int index = getIndex(it.value()); - if (index != 0) { - _free[oldtype].push_back(index); - } - // Insert value into new spot - if (_free[type].empty()) { - switch (type) { - case 0: it.setValue(getValue(type, _values0.size())); - _values0.push_back(val); - break; - case 1: it.setValue(getValue(type, _values1.size())); - _values1.push_back(T1(val)); - break; - case 2: it.setValue(getValue(type, _values2.size())); - _values2.push_back(T2(val)); - break; - case 3: it.setValue(getValue(type, _values3.size())); - _values3.push_back(T3(val)); - break; - default: assert(false); - } - } else { - it.setValue(getValue(type, _free[type].back())); - switch (type) { - case 0: _values0[_free[type].back()] = val; break; - case 1: _values1[_free[type].back()] = val; break; - case 2: _values2[_free[type].back()] = val; break; - case 3: _values3[_free[type].back()] = val; break; - default: assert(false); - } - _free[type].pop_back(); - } -} - -} // storage - diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.cpp b/storage/src/vespa/storage/bucketdb/lockablemap.cpp deleted file mode 100644 index cb5cbfbc203..00000000000 --- a/storage/src/vespa/storage/bucketdb/lockablemap.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "lockablemap.hpp" -#include "storagebucketinfo.h" -#include "judymultimap.h" - -namespace storage { - -uint8_t -getMinDiffBits(uint16_t minBits, const document::BucketId& a, const document::BucketId& b) { - for (uint32_t i = minBits; i <= std::min(a.getUsedBits(), b.getUsedBits()); i++) { - document::BucketId a1(i, a.getRawId()); - document::BucketId b1(i, b.getRawId()); - if (b1.getId() != a1.getId()) { - return i; - } - } - return minBits; -} - -bool -checkContains(document::BucketId::Type key, const document::BucketId& bucket, - document::BucketId& result, document::BucketId::Type& keyResult) -{ - document::BucketId id = document::BucketId(document::BucketId::keyToBucketId(key)); - if (id.contains(bucket)) { - result = id; - keyResult = key; - return true; - } - - return false; -} - -using bucketdb::StorageBucketInfo; - -template class LockableMap<storage::JudyMultiMap<StorageBucketInfo, StorageBucketInfo, StorageBucketInfo, StorageBucketInfo> >; - -} diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h deleted file mode 100644 index 168439786e9..00000000000 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * A map wrapper, adding locking to the map entries. It provides the - * following: - * - * - Guarantees thread safety. - * - Each returned value is given within a wrapper. As long as the - * wrapper for the value exist, this entry is locked in the map. - * This does not prevent other values from being used. Wrappers can - * be copied. Reference counting ensures value is locked until last - * wrapper copy dies. - * - Built in function for iterating taking a functor. Halts when - * encountering locked values. - */ -#pragma once - -#include "abstract_bucket_map.h" -#include <map> -#include <vespa/vespalib/util/printable.h> -#include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/document/bucket/bucketid.h> -#include <vespa/vespalib/util/time.h> -#include <mutex> -#include <condition_variable> -#include <cassert> - -namespace storage { - -template <typename Map> -class LockableMap final - : public bucketdb::AbstractBucketMap<typename Map::mapped_type> -{ -public: - using ParentType = bucketdb::AbstractBucketMap<typename Map::mapped_type>; - using WrappedEntry = typename ParentType::WrappedEntry; - using key_type = typename ParentType::key_type; - using mapped_type = typename ParentType::mapped_type; - using LockId = typename ParentType::LockId; - using EntryMap = typename ParentType::EntryMap; - using Decision = typename ParentType::Decision; - using BucketId = document::BucketId; - - LockableMap(); - ~LockableMap(); - bool operator==(const LockableMap& other) const; - bool operator!=(const LockableMap& other) const { - return ! (*this == other); - } - bool operator<(const LockableMap& other) const; - size_t size() const noexcept override; - size_t getMemoryUsage() const noexcept override; - vespalib::MemoryUsage detailed_memory_usage() const noexcept override; - bool empty() const noexcept override; - void swap(LockableMap&); - - WrappedEntry get(const key_type& key, const char* clientId, bool createIfNonExisting) override; - WrappedEntry get(const key_type& key, const char* clientId) { - return get(key, clientId, false); - } - - bool erase(const key_type& key, const char* clientId, bool haslock) override; - void insert(const key_type& key, const mapped_type& value, - const char* clientId, bool haslock, bool& preExisted) override; - - bool erase(const key_type& key, const char* clientId) - { return erase(key, clientId, false); } - void insert(const key_type& key, const mapped_type& value, - const char* clientId, bool& preExisted) - { return insert(key, value, clientId, false, preExisted); } - void clear(); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - /** - * Returns all buckets in the bucket database that can contain the given - * bucket. Usually, there should be only one such bucket, but in the case - * of inconsistent splitting, there may be more than one. - */ - EntryMap getContained(const BucketId& bucketId, const char* clientId) override; - - /** - * Returns all buckets in the bucket database that can contain the given - * bucket, and all buckets that that bucket contains. - */ - EntryMap getAll(const BucketId& bucketId, const char* clientId) override; - - /** - * Returns true iff bucket has no superbuckets or sub-buckets in the - * database. Usage assumption is that any operation that can cause the - * bucket to become inconsistent will require taking its lock, so by - * requiring the lock to be provided here we avoid race conditions. - */ - bool isConsistent(const WrappedEntry& entry) override; - - void showLockClients(vespalib::asciistream & out) const override; - -private: - struct hasher { - size_t operator () (const LockId & lid) const { return lid.hash(); } - }; - class LockIdSet : public vespalib::hash_set<LockId, hasher> { - typedef vespalib::hash_set<LockId, hasher> Hash; - public: - LockIdSet(); - ~LockIdSet(); - void print(std::ostream& out, bool verbose, const std::string& indent) const; - bool exist(const LockId& lid) const { return this->find(lid) != Hash::end(); } - size_t getMemoryUsage() const; - }; - - class LockWaiters { - typedef vespalib::hash_map<size_t, LockId> WaiterMap; - public: - typedef size_t Key; - typedef typename WaiterMap::const_iterator const_iterator; - LockWaiters(); - ~LockWaiters(); - Key insert(const LockId & lid); - void erase(Key id) { _map.erase(id); } - const_iterator begin() const { return _map.begin(); } - const_iterator end() const { return _map.end(); } - private: - Key _id; - WaiterMap _map; - }; - - class ReadGuardImpl; - - Map _map; - mutable std::mutex _lock; - std::condition_variable _cond; - LockIdSet _lockedKeys; - LockWaiters _lockWaiters; - - void unlock(const key_type& key) override; - bool findNextKey(key_type& key, mapped_type& val, const char* clientId, - std::unique_lock<std::mutex> &guard); - bool handleDecision(key_type& key, mapped_type& val, Decision decision); - void acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard); - - void do_for_each_mutable_unordered(std::function<Decision(uint64_t, mapped_type&)> func, - const char* clientId) override; - - void do_for_each(std::function<Decision(uint64_t, const mapped_type&)> func, - const char* clientId, - const key_type& first, - const key_type& last) override; - - void do_for_each_chunked(std::function<Decision(uint64_t, const mapped_type&)> func, - const char* clientId, - vespalib::duration yieldTime, - uint32_t chunkSize) override; - - std::unique_ptr<bucketdb::ReadGuard<typename Map::mapped_type>> do_acquire_read_guard() const override; - - /** - * Process up to `chunkSize` bucket database entries from--and possibly - * including--the bucket pointed to by `key`. - * - * Returns true if additional chunks may be processed after the call to - * this function has returned, false if iteration has completed or if - * `functor` returned an abort-decision. - * - * Modifies `key` in-place to point to the next key to process for the next - * invocation of this function. - */ - bool processNextChunk(std::function<Decision(uint64_t, const mapped_type&)>& func, - key_type& key, - const char* clientId, - uint32_t chunkSize); - - /** - * Returns the given bucket, its super buckets and its sub buckets. - */ - void getAllWithoutLocking(const BucketId& bucket, - std::vector<BucketId::Type>& keys); - - /** - * Retrieves the most specific bucket id (highest used bits) that matches - * the given bucket. - * - * If a match is found, result is set to the bucket id found, and keyResult - * is set to the corresponding key (reversed) - * - * If not found, nextKey is set to the key after one that could have - * matched and we return false. - */ - bool getMostSpecificMatch(const BucketId& bucket, - BucketId& result, - BucketId::Type& keyResult, - BucketId::Type& nextKey); - - /** - * Finds all buckets that can contain the given bucket, except for the - * bucket itself (that is, its super buckets) - */ - void getAllContaining(const BucketId& bucket, - std::vector<BucketId::Type>& keys); - - /** - * Find the given list of keys in the map and add them to the map of - * results, locking them in the process. - */ - void addAndLockResults(const std::vector<BucketId::Type>& keys, - const char* clientId, - std::map<BucketId, WrappedEntry>& results, - std::unique_lock<std::mutex> &guard); -}; - -} // storage - diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp deleted file mode 100644 index f6ae420cb34..00000000000 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ /dev/null @@ -1,665 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "lockablemap.h" -#include <vespa/vespalib/util/hdr_abort.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/vespalib/stllike/hash_set.hpp> -#include <thread> -#include <chrono> -#include <ostream> - -namespace storage { - -template<typename Map> -LockableMap<Map>::LockIdSet::LockIdSet() : Hash() { } - -template<typename Map> -LockableMap<Map>::LockIdSet::~LockIdSet() = default; - -template<typename Map> -size_t -LockableMap<Map>::LockIdSet::getMemoryUsage() const { - return Hash::getMemoryConsumption(); -} - -template<typename Map> -LockableMap<Map>::LockWaiters::LockWaiters() : _id(0), _map() { } - -template<typename Map> -LockableMap<Map>::LockWaiters::~LockWaiters() = default; - -template<typename Map> -size_t -LockableMap<Map>::LockWaiters::insert(const LockId & lid) { - Key id(_id++); - _map.insert(typename WaiterMap::value_type(id, lid)); - return id; -} - -template<typename Map> -LockableMap<Map>::LockableMap() - : _map(), - _lock(), - _cond(), - _lockedKeys(), - _lockWaiters() -{} - -template<typename Map> -LockableMap<Map>::~LockableMap() = default; - -template<typename Map> -bool -LockableMap<Map>::operator==(const LockableMap<Map>& other) const -{ - std::lock_guard<std::mutex> guard(_lock); - std::lock_guard<std::mutex> guard2(other._lock); - return (_map == other._map); -} - -template<typename Map> -bool -LockableMap<Map>::operator<(const LockableMap<Map>& other) const -{ - std::lock_guard<std::mutex> guard(_lock); - std::lock_guard<std::mutex> guard2(other._lock); - return (_map < other._map); -} - -template<typename Map> -size_t -LockableMap<Map>::size() const noexcept -{ - std::lock_guard<std::mutex> guard(_lock); - return _map.size(); -} - -template<typename Map> -size_t -LockableMap<Map>::getMemoryUsage() const noexcept -{ - std::lock_guard<std::mutex> guard(_lock); - return _map.getMemoryUsage() + _lockedKeys.getMemoryUsage() + - sizeof(std::mutex) + sizeof(std::condition_variable); -} - -template <typename Map> -vespalib::MemoryUsage LockableMap<Map>::detailed_memory_usage() const noexcept { - // We don't have any details for this map type, just count everything as "allocated". - size_t used = getMemoryUsage(); - vespalib::MemoryUsage mem_usage; - mem_usage.incAllocatedBytes(used); - mem_usage.incUsedBytes(used); - return mem_usage; -} - -template<typename Map> -bool -LockableMap<Map>::empty() const noexcept -{ - std::lock_guard<std::mutex> guard(_lock); - return _map.empty(); -} - -template<typename Map> -void -LockableMap<Map>::swap(LockableMap<Map>& other) -{ - std::lock_guard<std::mutex> guard(_lock); - std::lock_guard<std::mutex> guard2(other._lock); - return _map.swap(other._map); -} - -template<typename Map> -void LockableMap<Map>::acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard) -{ - if (_lockedKeys.exist(lid)) { - typename LockWaiters::Key waitId(_lockWaiters.insert(lid)); - while (_lockedKeys.exist(lid)) { - _cond.wait(guard); - } - _lockWaiters.erase(waitId); - } -} - -template<typename Map> -typename LockableMap<Map>::WrappedEntry -LockableMap<Map>::get(const key_type& key, const char* clientId, bool createIfNonExisting) -{ - LockId lid(key, clientId); - std::unique_lock<std::mutex> guard(_lock); - acquireKey(lid, guard); - bool preExisted = false; - typename Map::iterator it = - _map.find(key, createIfNonExisting, preExisted); - - if (it == _map.end()) { - return WrappedEntry(); - } - _lockedKeys.insert(lid); - return WrappedEntry(*this, key, it->second, clientId, preExisted); -} - -template<typename Map> -bool -LockableMap<Map>::erase(const key_type& key, const char* client_id, bool has_lock) -{ - LockId lid(key, client_id); - std::unique_lock<std::mutex> guard(_lock); - if (!has_lock) { - acquireKey(lid, guard); - } - return _map.erase(key); -} - -template<typename Map> -void -LockableMap<Map>::insert(const key_type& key, const mapped_type& value, - const char* client_id, bool has_lock, bool& preExisted) -{ - LockId lid(key, client_id); - std::unique_lock<std::mutex> guard(_lock); - if (!has_lock) { - acquireKey(lid, guard); - } - _map.insert(key, value, preExisted); -} - -template<typename Map> -void -LockableMap<Map>::clear() -{ - std::lock_guard<std::mutex> guard(_lock); - _map.clear(); -} - -template<typename Map> -bool -LockableMap<Map>::findNextKey(key_type& key, mapped_type& val, - const char* clientId, - std::unique_lock<std::mutex> &guard) -{ - // Wait for next value to unlock. - typename Map::iterator it(_map.lower_bound(key)); - while (it != _map.end() && _lockedKeys.exist(LockId(it->first, ""))) { - auto wait_id = _lockWaiters.insert(LockId(it->first, clientId)); - _cond.wait(guard); - _lockWaiters.erase(wait_id); - it = _map.lower_bound(key); - } - if (it == _map.end()) { - return true; - } - key = it->first; - val = it->second; - return false; -} - -template<typename Map> -bool -LockableMap<Map>::handleDecision(key_type& key, mapped_type& val, - Decision decision) -{ - bool b; - switch (decision) { - case Decision::UPDATE: - _map.insert(key, val, b); - break; - case Decision::REMOVE: - _map.erase(key); - break; - case Decision::ABORT: - return true; - case Decision::CONTINUE: - break; - default: - HDR_ABORT("should not be reached"); - } - return false; -} - -template<typename Map> -void LockableMap<Map>::do_for_each_mutable_unordered(std::function<Decision(uint64_t, mapped_type&)> func, - const char* clientId) -{ - key_type key = 0; - mapped_type val; - std::unique_lock<std::mutex> guard(_lock); - while (true) { - if (findNextKey(key, val, clientId, guard)) { - return; - } - Decision d(func(const_cast<const key_type&>(key), val)); - if (handleDecision(key, val, d)) { - return; - } - ++key; - } -} - -template<typename Map> -void LockableMap<Map>::do_for_each(std::function<Decision(uint64_t, const mapped_type&)> func, - const char* clientId, - const key_type& first, - const key_type& last) -{ - key_type key = first; - mapped_type val; - std::unique_lock<std::mutex> guard(_lock); - while (true) { - if (findNextKey(key, val, clientId, guard) || key > last) { - return; - } - Decision d(func(const_cast<const key_type&>(key), val)); - assert(d == Decision::ABORT || d == Decision::CONTINUE); - if (handleDecision(key, val, d)) { - return; - } - ++key; - } -} - -template <typename Map> -bool -LockableMap<Map>::processNextChunk(std::function<Decision(uint64_t, const mapped_type&)>& func, - key_type& key, - const char* clientId, - const uint32_t chunkSize) -{ - mapped_type val; - std::unique_lock<std::mutex> guard(_lock); - for (uint32_t processed = 0; processed < chunkSize; ++processed) { - if (findNextKey(key, val, clientId, guard)) { - return false; - } - Decision d(func(const_cast<const key_type&>(key), val)); - if (handleDecision(key, val, d)) { - return false; - } - ++key; - } - return true; -} - -template <typename Map> -void LockableMap<Map>::do_for_each_chunked(std::function<Decision(uint64_t, const mapped_type&)> func, - const char* clientId, - vespalib::duration yieldTime, - uint32_t chunkSize) -{ - key_type key{}; - while (processNextChunk(func, key, clientId, chunkSize)) { - // Rationale: delay iteration for as short a time as possible while - // allowing another thread blocked on the main DB mutex to acquire it - // in the meantime. Simply yielding the thread does not have the - // intended effect with the Linux scheduler. - // This is a pragmatic stop-gap solution; a more robust change requires - // the redesign of bucket DB locking and signalling semantics in the - // face of blocked point lookups. - std::this_thread::sleep_for(yieldTime); - } -} - -// TODO This is a placeholder that has to work around the const-ness and type quirks of -// the legacy LockableMap implementation. In particular, it offers no snapshot isolation -// at all, nor does it support the "get parents and self" bucket lookup operation. -template <typename Map> -class LockableMap<Map>::ReadGuardImpl final - : public bucketdb::ReadGuard<typename Map::mapped_type> -{ - const LockableMap<Map>& _map; -public: - using mapped_type = typename Map::mapped_type; - - explicit ReadGuardImpl(const LockableMap<Map>& map) : _map(map) {} - ~ReadGuardImpl() override = default; - - std::vector<mapped_type> find_parents_and_self(const document::BucketId&) const override { - abort(); // Finding just parents+self isn't supported by underlying legacy DB API! - } - - std::vector<mapped_type> find_parents_self_and_children(const document::BucketId& bucket) const override { - auto& mutable_map = const_cast<LockableMap<Map>&>(_map); // _map is thread safe. - auto locked_entries = mutable_map.getAll(bucket, "ReadGuardImpl::find_parents_self_and_children"); - std::vector<mapped_type> entries; - entries.reserve(locked_entries.size()); - for (auto& e : locked_entries) { - entries.emplace_back(*e.second); - } - return entries; - } - - void for_each(std::function<void(uint64_t, const mapped_type&)> func) const override { - auto decision_wrapper = [&func](uint64_t key, const mapped_type& value) -> Decision { - func(key, value); - return Decision::CONTINUE; - }; - auto& mutable_map = const_cast<LockableMap<Map>&>(_map); // _map is thread safe. - mutable_map.for_each_chunked(std::move(decision_wrapper), "ReadGuardImpl::for_each"); - } - - [[nodiscard]] uint64_t generation() const noexcept override { - return 0; - } -}; - -template <typename Map> -std::unique_ptr<bucketdb::ReadGuard<typename Map::mapped_type>> -LockableMap<Map>::do_acquire_read_guard() const { - return std::make_unique<ReadGuardImpl>(*this); -} - -template<typename Map> -void -LockableMap<Map>::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - std::lock_guard<std::mutex> guard(_lock); - out << "LockableMap {\n" << indent << " "; - - if (verbose) { - for (const auto entry : _map) { - out << "Key: " << BucketId(BucketId::keyToBucketId(entry.first)) - << " Value: " << entry.second << "\n" << indent << " "; - } - - out << "\n" << indent << " Locked keys: "; - _lockedKeys.print(out, verbose, indent + " "); - } - out << "} : "; - - out << _map; -} - -template<typename Map> -void -LockableMap<Map>::LockIdSet::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "hash {"; - for (typename Hash::const_iterator it(Hash::begin()), mt(Hash::end()); it != mt; it++) { - if (verbose) { - out << "\n" << indent << " "; - } else { - out << " "; - } - - out << *it; - } - if (verbose) out << "\n" << indent; - out << " }"; -} - -template<typename Map> -void -LockableMap<Map>::unlock(const key_type& key) -{ - std::lock_guard<std::mutex> guard(_lock); - _lockedKeys.erase(LockId(key, "")); - _cond.notify_all(); -} - -/** - * Check whether the given key contains the given bucket. - * Sets result to the bucket corresponding to the key, and keyResult - * to the key if true. - */ -bool -checkContains(document::BucketId::Type key, const document::BucketId& bucket, - document::BucketId& result, document::BucketId::Type& keyResult); - -/** - * Retrieves the most specific bucket id (highest used bits) that contains - * the given bucket. - * - * If a match is found, result is set to the bucket id found, and keyResult is - * set to the corresponding key (reversed) - * - * If not found, nextKey is set to the key after one that could have matched - * and we return false. - */ -template<typename Map> -bool -LockableMap<Map>::getMostSpecificMatch(const BucketId& bucket, - BucketId& result, - BucketId::Type& keyResult, - BucketId::Type& nextKey) -{ - typename Map::const_iterator iter = _map.lower_bound(bucket.toKey()); - - nextKey = 0; - - // We should now have either the bucket we are looking for - // (if the exact bucket exists), or one right after. - if (iter != _map.end()) { - nextKey = iter->first; - - if (checkContains(iter->first, bucket, result, keyResult)) { - return true; - } - } - - if (iter != _map.begin()) { - --iter; // If iter was map.end(), we should now end up at the last item in the map - nextKey = iter->first; - - if (checkContains(iter->first, bucket, result, keyResult)) { - return true; - } - } - - return false; -} - -/** - * Finds all buckets that can contain the given bucket, except for the bucket - * itself. - */ -template<typename Map> -void -LockableMap<Map>::getAllContaining(const BucketId& bucket, - std::vector<BucketId::Type>& keys) -{ - BucketId id = bucket; - - // Find other buckets that contain this bucket. - // TODO: Optimize? - while (id.getUsedBits() > 1) { - id.setUsedBits(id.getUsedBits() - 1); - id = id.stripUnused(); - BucketId::Type key = id.toKey(); - - typename Map::const_iterator iter = _map.find(key); - if (iter != _map.end()) { - keys.push_back(key); - } - } -} - -template<typename Map> -void -LockableMap<Map>::addAndLockResults( - const std::vector<BucketId::Type>& keys, - const char* clientId, - std::map<BucketId, WrappedEntry>& results, - std::unique_lock<std::mutex> &guard) -{ - // Wait until all buckets are free to be added, then add them all. - while (true) { - bool allOk = true; - key_type waitingFor(0); - - for (uint32_t i=0; i<keys.size(); i++) { - if (_lockedKeys.exist(LockId(keys[i], clientId))) { - waitingFor = keys[i]; - allOk = false; - break; - } - } - - if (!allOk) { - typename LockWaiters::Key waitId(_lockWaiters.insert(LockId(waitingFor, clientId))); - _cond.wait(guard); - _lockWaiters.erase(waitId); - } else { - for (uint32_t i=0; i<keys.size(); i++) { - typename Map::iterator it = _map.find(keys[i]); - if (it != _map.end()) { - _lockedKeys.insert(LockId(keys[i], clientId)); - results[BucketId(BucketId::keyToBucketId(keys[i]))] - = WrappedEntry(*this, keys[i], it->second, - clientId, true); - } - } - break; - } - } -} - -uint8_t getMinDiffBits(uint16_t minBits, const document::BucketId& a, const document::BucketId& b); - -template<typename Map> -typename LockableMap<Map>::EntryMap -LockableMap<Map>::getContained(const BucketId& bucket, - const char* clientId) -{ - std::unique_lock<std::mutex> guard(_lock); - std::map<BucketId, WrappedEntry> results; - - BucketId result; - BucketId::Type keyResult; - BucketId::Type nextKey; - - std::vector<BucketId::Type> keys; - - if (getMostSpecificMatch(bucket, result, keyResult, nextKey)) { - keys.push_back(keyResult); - - // Find the super buckets for the most specific match - getAllContaining(result, keys); - } else { - // Find the super buckets for the input bucket - // because getMostSpecificMatch() might not find the most specific - // match in all cases of inconsistently split buckets - getAllContaining(bucket, keys); - } - - if (!keys.empty()) { - addAndLockResults(keys, clientId, results, guard); - } - - return results; -} - -template<typename Map> -void -LockableMap<Map>::getAllWithoutLocking(const BucketId& bucket, - std::vector<BucketId::Type>& keys) -{ - BucketId result; - BucketId::Type keyResult; - BucketId::Type nextKey; - - typename Map::iterator it = _map.end(); - - if (getMostSpecificMatch(bucket, result, keyResult, nextKey)) { - keys.push_back(keyResult); - - // Find the super buckets for the most specific match - getAllContaining(result, keys); - - it = _map.find(keyResult); - if (it != _map.end()) { - // Skipping nextKey, since it was equal to keyResult - ++it; - } - } else { - // Find the super buckets for the input bucket - // because getMostSpecificMatch() might not find the most specific - // match in all cases of inconsistently split buckets - getAllContaining(bucket, keys); - - it = _map.find(nextKey); - if (it != _map.end()) { - // Nextkey might be contained in the imput bucket, - // e.g. if it is the first bucket in bucketdb - BucketId id = BucketId(BucketId::keyToBucketId(it->first)); - if (!bucket.contains(id)) { - ++it; - } - } - } - - // Buckets contained in the found bucket will come immediately after it. - // Traverse the map to find them. - for (; it != _map.end(); ++it) { - BucketId id(BucketId(BucketId::keyToBucketId(it->first))); - - if (bucket.contains(id)) { - keys.push_back(it->first); - } else { - break; - } - } -} - -/** - * Returns the given bucket, its super buckets and its sub buckets. - */ -template<typename Map> -typename LockableMap<Map>::EntryMap -LockableMap<Map>::getAll(const BucketId& bucket, const char* clientId) -{ - std::unique_lock<std::mutex> guard(_lock); - - std::map<BucketId, WrappedEntry> results; - std::vector<BucketId::Type> keys; - - getAllWithoutLocking(bucket, keys); - - addAndLockResults(keys, clientId, results, guard); - - return results; -} - -template<typename Map> -bool -LockableMap<Map>::isConsistent(const typename LockableMap<Map>::WrappedEntry& entry) -{ - std::lock_guard<std::mutex> guard(_lock); - - std::vector<BucketId::Type> keys; - - getAllWithoutLocking(entry.getBucketId(), keys); - assert(keys.size() >= 1); - assert(keys.size() != 1 || keys[0] == entry.getKey()); - - return keys.size() == 1; -} - -template<typename Map> -void -LockableMap<Map>::showLockClients(vespalib::asciistream & out) const -{ - std::lock_guard<std::mutex> guard(_lock); - out << "Currently grabbed locks:"; - for (typename LockIdSet::const_iterator it = _lockedKeys.begin(); - it != _lockedKeys.end(); ++it) - { - out << "\n " - << BucketId(BucketId::keyToBucketId(it->_key)) - << " - " << it->_owner; - } - out << "\nClients waiting for keys:"; - for (typename LockWaiters::const_iterator it = _lockWaiters.begin(); - it != _lockWaiters.end(); ++it) - { - out << "\n " - << BucketId(BucketId::keyToBucketId(it->second._key)) - << " - " << it->second._owner; - } -} - -} diff --git a/storage/src/vespa/storage/bucketdb/storbucketdb.cpp b/storage/src/vespa/storage/bucketdb/storbucketdb.cpp index 9e692d7d602..b6a5a2320b1 100644 --- a/storage/src/vespa/storage/bucketdb/storbucketdb.cpp +++ b/storage/src/vespa/storage/bucketdb/storbucketdb.cpp @@ -2,8 +2,6 @@ #include "storbucketdb.h" #include "btree_lockable_map.h" -#include "judymultimap.hpp" -#include "lockablemap.h" #include <vespa/log/log.h> LOG_SETUP(".storage.bucketdb.stor_bucket_db"); @@ -40,10 +38,6 @@ operator<<(std::ostream& out, const StorageBucketInfo& info) { namespace { -std::unique_ptr<AbstractBucketMap<StorageBucketInfo>> make_legacy_db_impl() { - return std::make_unique<LockableMap<JudyMultiMap<StorageBucketInfo>>>(); -} - std::unique_ptr<AbstractBucketMap<StorageBucketInfo>> make_btree_db_impl() { return std::make_unique<BTreeLockableMap<StorageBucketInfo>>(); } @@ -52,9 +46,8 @@ std::unique_ptr<AbstractBucketMap<StorageBucketInfo>> make_btree_db_impl() { } // bucketdb -StorBucketDatabase::StorBucketDatabase(bool use_btree_db) - : _impl(use_btree_db ? bucketdb::make_btree_db_impl() - : bucketdb::make_legacy_db_impl()) +StorBucketDatabase::StorBucketDatabase([[maybe_unused]] bool use_btree_db) + : _impl(bucketdb::make_btree_db_impl()) {} void @@ -142,6 +135,4 @@ StorBucketDatabase::acquire_read_guard() const { return _impl->acquire_read_guard(); } -template class JudyMultiMap<bucketdb::StorageBucketInfo>; - } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 2653391ecfa..8e1ee6cbb3a 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -2,7 +2,6 @@ #include "filestorhandlerimpl.h" #include "filestormanager.h" -#include <vespa/storage/bucketdb/lockablemap.hpp> #include <vespa/storage/bucketdb/minimumusedbitstracker.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/content_bucket_space_repo.h> @@ -18,8 +17,10 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> +#include <thread> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".persistence.filestor.manager"); diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp index 0885483f674..21cf8a064e1 100644 --- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp +++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/programoptions.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/storage/bucketdb/judyarray.h> #include <vespa/vespalib/util/stringfmt.h> #include <iomanip> #include <iostream> |