aboutsummaryrefslogtreecommitdiffstats
path: root/storage/src
diff options
context:
space:
mode:
authorTor Brede Vekterli <vekterli@verizonmedia.com>2020-10-28 08:52:57 +0000
committerTor Brede Vekterli <vekterli@verizonmedia.com>2020-10-28 08:52:57 +0000
commit68ca16661cb5a35818f098f325b317a542d55a16 (patch)
treea4bb65b33d171cb5ed4dd89569ce23a361252d62 /storage/src
parent4d64cbe4e531cb7be1061f1f54809d1d0a1b0061 (diff)
Remove legacy Judy array-backed bucket DB implementation
Diffstat (limited to 'storage/src')
-rw-r--r--storage/src/tests/bucketdb/CMakeLists.txt2
-rw-r--r--storage/src/tests/bucketdb/judyarraytest.cpp201
-rw-r--r--storage/src/tests/bucketdb/judymultimaptest.cpp152
-rw-r--r--storage/src/tests/bucketdb/lockablemaptest.cpp6
-rw-r--r--storage/src/vespa/storage/bucketdb/CMakeLists.txt2
-rw-r--r--storage/src/vespa/storage/bucketdb/bucketmanager.cpp2
-rw-r--r--storage/src/vespa/storage/bucketdb/judyarray.cpp146
-rw-r--r--storage/src/vespa/storage/bucketdb/judyarray.h211
-rw-r--r--storage/src/vespa/storage/bucketdb/judymultimap.h179
-rw-r--r--storage/src/vespa/storage/bucketdb/judymultimap.hpp406
-rw-r--r--storage/src/vespa/storage/bucketdb/lockablemap.cpp38
-rw-r--r--storage/src/vespa/storage/bucketdb/lockablemap.h212
-rw-r--r--storage/src/vespa/storage/bucketdb/lockablemap.hpp665
-rw-r--r--storage/src/vespa/storage/bucketdb/storbucketdb.cpp13
-rw-r--r--storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp3
-rw-r--r--storage/src/vespa/storage/tools/generatedistributionbits.cpp1
16 files changed, 7 insertions, 2232 deletions
diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt
index 2acaea1940d..81c65ec9ad3 100644
--- a/storage/src/tests/bucketdb/CMakeLists.txt
+++ b/storage/src/tests/bucketdb/CMakeLists.txt
@@ -5,8 +5,6 @@ vespa_add_executable(storage_bucketdb_gtest_runner_app TEST
bucketinfotest.cpp
bucketmanagertest.cpp
gtest_runner.cpp
- judyarraytest.cpp
- judymultimaptest.cpp
lockablemaptest.cpp
DEPENDS
storage
diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp
deleted file mode 100644
index 72967e8b4c0..00000000000
--- a/storage/src/tests/bucketdb/judyarraytest.cpp
+++ /dev/null
@@ -1,201 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storage/bucketdb/judyarray.h>
-#include <boost/random.hpp>
-#include <vespa/vespalib/gtest/gtest.h>
-#include <gmock/gmock.h>
-#include <map>
-#include <vector>
-
-using namespace ::testing;
-
-namespace storage {
-
-namespace {
- std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>>
- getJudyArrayContents(const JudyArray& array) {
- std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> vals;
- for (JudyArray::const_iterator it = array.begin();
- it != array.end(); ++it)
- {
- vals.push_back(std::make_pair(it.key(), it.value()));
- }
- return vals;
- }
-}
-
-TEST(JudyArrayTest, iterating) {
- JudyArray array;
- // Test that things are sane for empty document
- ASSERT_EQ(array.begin(), array.end());
- // Add some values
- std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values({
- {3, 2}, {5, 12}, {15, 8}, {13, 10}, {7, 6}, {9, 4}
- });
- for (uint32_t i=0; i<values.size(); ++i) {
- array.insert(values[i].first, values[i].second);
- }
- // Create expected result
- std::sort(values.begin(), values.end());
- // Test that we can iterate through const iterator
- auto foundVals = getJudyArrayContents(array);
- ASSERT_EQ(values, foundVals);
-
- { // Test that we can alter through non-const iterator
- JudyArray::iterator it = array.begin();
- ++it;
- ++it;
- it.setValue(20);
- ASSERT_EQ((JudyArray::key_type) 7, it.key());
- ASSERT_EQ((JudyArray::data_type) 20, array[7]);
- it.remove();
- ASSERT_EQ((JudyArray::size_type) 5, getJudyArrayContents(array).size());
- ASSERT_EQ(array.end(), array.find(7));
- values.erase(values.begin() + 2);
- ASSERT_EQ(values, getJudyArrayContents(array));
- // And that we can continue iterating after removing.
- ++it;
- ASSERT_EQ((JudyArray::key_type) 9, it.key());
- ASSERT_EQ((JudyArray::data_type) 4, array[9]);
- }
- { // Test printing of iterators
- JudyArray::ConstIterator cit = array.begin();
- EXPECT_THAT(cit.toString(), MatchesRegex("^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$"));
- JudyArray::Iterator it = array.end();
- EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: (0x)?0\\)$"));
- }
-}
-
-TEST(JudyArrayTest, dual_array_functions) {
- JudyArray array1;
- JudyArray array2;
- // Add values to array1
- std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values1({
- {3, 2}, {5, 12}, {15, 8}, {13, 10}, {7, 6}, {9, 4}
- });
- for (uint32_t i=0; i<values1.size(); ++i) {
- array1.insert(values1[i].first, values1[i].second);
- }
- // Add values to array2
- std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values2({
- {4, 5}, {9, 40}
- });
- for (uint32_t i=0; i<values2.size(); ++i) {
- array2.insert(values2[i].first, values2[i].second);
- }
- // Create expected result
- std::sort(values1.begin(), values1.end());
- std::sort(values2.begin(), values2.end());
-
- EXPECT_EQ(values1, getJudyArrayContents(array1));
- EXPECT_EQ(values2, getJudyArrayContents(array2));
- EXPECT_LT(array2, array1);
- EXPECT_NE(array1, array2);
- array1.swap(array2);
- EXPECT_EQ(values1, getJudyArrayContents(array2));
- EXPECT_EQ(values2, getJudyArrayContents(array1));
- EXPECT_LT(array1, array2);
- EXPECT_NE(array1, array2);
-
- // Test some operators
- JudyArray array3;
- for (uint32_t i=0; i<values1.size(); ++i) {
- array3.insert(values1[i].first, values1[i].second);
- }
- EXPECT_NE(array1, array3);
- EXPECT_EQ(array2, array3);
- EXPECT_FALSE(array2 < array3);
-}
-
-TEST(JudyArrayTest, size) {
- JudyArray array;
- EXPECT_EQ(array.begin(), array.end());
- EXPECT_TRUE(array.empty());
- EXPECT_EQ((JudyArray::size_type) 0, array.size());
- EXPECT_EQ((JudyArray::size_type) 0, array.getMemoryUsage());
-
- // Test each method one can insert stuff into array
- array.insert(4, 3);
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- array.insert(4, 7);
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- EXPECT_EQ((JudyArray::size_type) 24, array.getMemoryUsage());
-
- array[6] = 8;
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- array[6] = 10;
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- EXPECT_EQ((JudyArray::size_type) 40, array.getMemoryUsage());
-
- bool preExisted;
- array.find(8, true, preExisted);
- EXPECT_EQ(false, preExisted);
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- array.find(8, true, preExisted);
- EXPECT_EQ(true, preExisted);
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- EXPECT_EQ((JudyArray::size_type) 3, array.size());
- EXPECT_EQ((JudyArray::size_type) 56, array.getMemoryUsage());
-
- // Test each method one can remove stuff in array with
- array.erase(8);
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- array.erase(8);
- EXPECT_EQ(getJudyArrayContents(array).size(), array.size());
- EXPECT_EQ((JudyArray::size_type) 2, array.size());
- EXPECT_EQ((JudyArray::size_type) 40, array.getMemoryUsage());
-}
-
-TEST(JudyArrayTest, stress) {
- // Do a lot of random stuff to both judy array and std::map. Ensure equal
- // behaviour
-
- JudyArray judyArray;
- typedef std::map<JudyArray::key_type, JudyArray::data_type> StdMap;
- StdMap stdMap;
-
- boost::rand48 rnd(55);
-
- for (uint32_t checkpoint=0; checkpoint<50; ++checkpoint) {
- for (uint32_t opnr=0; opnr<500; ++opnr) {
- int optype = rnd() % 100;
- if (optype < 30) { // Insert
- JudyArray::key_type key(rnd() % 500);
- JudyArray::key_type value(rnd());
- judyArray.insert(key, value);
- stdMap[key] = value;
- } else if (optype < 50) { // operator[]
- JudyArray::key_type key(rnd() % 500);
- JudyArray::key_type value(rnd());
- judyArray[key] = value;
- stdMap[key] = value;
- } else if (optype < 70) { // erase()
- JudyArray::key_type key(rnd() % 500);
- EXPECT_EQ(stdMap.erase(key), judyArray.erase(key));
- } else if (optype < 75) { // size()
- EXPECT_EQ(stdMap.size(), judyArray.size());
- } else if (optype < 78) { // empty()
- EXPECT_EQ(stdMap.empty(), judyArray.empty());
- } else { // find()
- JudyArray::key_type key(rnd() % 500);
- auto it = judyArray.find(key);
- auto it2 = stdMap.find(key);
- EXPECT_EQ(it2 == stdMap.end(), it == judyArray.end());
- if (it != judyArray.end()) {
- EXPECT_EQ(it.key(), it2->first);
- EXPECT_EQ(it.value(), it2->second);
- }
- }
- }
- // Ensure judy array contents is equal to std::map's at this point
- StdMap tmpMap;
- for (JudyArray::const_iterator it = judyArray.begin();
- it != judyArray.end(); ++it)
- {
- tmpMap[it.key()] = it.value();
- }
- EXPECT_EQ(stdMap, tmpMap);
- }
-}
-
-} // storage
diff --git a/storage/src/tests/bucketdb/judymultimaptest.cpp b/storage/src/tests/bucketdb/judymultimaptest.cpp
deleted file mode 100644
index 254dbb78b18..00000000000
--- a/storage/src/tests/bucketdb/judymultimaptest.cpp
+++ /dev/null
@@ -1,152 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/storage/bucketdb/judymultimap.h>
-#include <vespa/storage/bucketdb/judymultimap.hpp>
-#include <vespa/vespalib/gtest/gtest.h>
-#include <map>
-#include <ostream>
-#include <vector>
-
-using namespace ::testing;
-
-namespace storage {
-
-namespace {
- struct B;
- struct C;
-
- struct A {
- int _val1;
- int _val2;
- int _val3;
-
- A() = default;
- A(const B &);
- A(const C &);
- A(int val1, int val2, int val3)
- : _val1(val1), _val2(val2), _val3(val3) {}
-
- static bool mayContain(const A&) { return true; }
-
- bool operator==(const A& a) const {
- return (_val1 == a._val1 && _val2 == a._val2 && _val3 == a._val3);
- }
- };
-
- struct B {
- int _val1;
- int _val2;
-
- B() = default;
- B(const A& a) : _val1(a._val1), _val2(a._val2) {}
- B(int val1, int val2) : _val1(val1), _val2(val2) {}
-
- static bool mayContain(const A& a) { return (a._val3 == 0); }
- };
-
- struct C {
- int _val1;
-
- C() = default;
- C(const A& a) : _val1(a._val1) {}
- C(int val1) : _val1(val1) {}
-
- static bool mayContain(const A& a) { return (a._val2 == 0 && a._val3 == 0); }
- };
-
- A::A(const B& b) : _val1(b._val1), _val2(b._val2), _val3(0) {}
- A::A(const C& c) : _val1(c._val1), _val2(0), _val3(0) {}
-
- std::ostream& operator<<(std::ostream& out, const A& a) {
- return out << "A(" << a._val1 << ", " << a._val2 << ", " << a._val3 << ")";
- }
- std::ostream& operator<<(std::ostream& out, const B& b) {
- return out << "B(" << b._val1 << ", " << b._val2 << ")";
- }
- std::ostream& operator<<(std::ostream& out, const C& c) {
- return out << "C(" << c._val1 << ")";
- }
-}
-
-TEST(JudyMultiMapTest, simple_usage) {
- typedef JudyMultiMap<C, B, A> MultiMap;
- MultiMap multiMap;
- // Do some insertions
- bool preExisted;
- EXPECT_TRUE(multiMap.empty());
- multiMap.insert(16, A(1, 2, 3), preExisted);
- EXPECT_EQ(false, preExisted);
- multiMap.insert(11, A(4, 6, 0), preExisted);
- EXPECT_EQ(false, preExisted);
- multiMap.insert(14, A(42, 0, 0), preExisted);
- EXPECT_EQ(false, preExisted);
- EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()) << multiMap.toString();
-
- multiMap.insert(11, A(4, 7, 0), preExisted);
- EXPECT_EQ(true, preExisted);
- EXPECT_EQ((MultiMap::size_type) 3, multiMap.size());
- EXPECT_FALSE(multiMap.empty());
-
- // Access some elements
- EXPECT_EQ(A(4, 7, 0), multiMap[11]);
- EXPECT_EQ(A(1, 2, 3), multiMap[16]);
- EXPECT_EQ(A(42,0, 0), multiMap[14]);
-
- // Do removes
- EXPECT_EQ(multiMap.erase(12), 0);
- EXPECT_EQ((MultiMap::size_type) 3, multiMap.size());
-
- EXPECT_EQ(multiMap.erase(14), 1);
- EXPECT_EQ((MultiMap::size_type) 2, multiMap.size());
-
- EXPECT_EQ(multiMap.erase(11), 1);
- EXPECT_EQ(multiMap.erase(16), 1);
- EXPECT_EQ((MultiMap::size_type) 0, multiMap.size());
- EXPECT_TRUE(multiMap.empty());
-}
-
-TEST(JudyMultiMapTest, iterator) {
- typedef JudyMultiMap<C, B, A> MultiMap;
- MultiMap multiMap;
- bool preExisted;
- // Do some insertions
- multiMap.insert(16, A(1, 2, 3), preExisted);
- multiMap.insert(11, A(4, 6, 0), preExisted);
- multiMap.insert(14, A(42, 0, 0), preExisted);
-
- MultiMap::Iterator iter = multiMap.begin();
- EXPECT_EQ((uint64_t)11, (uint64_t)iter.key());
- EXPECT_EQ(A(4, 6, 0), iter.value());
- ++iter;
- EXPECT_EQ((uint64_t)14, (uint64_t)iter.key());
- EXPECT_EQ(A(42, 0, 0), iter.value());
- ++iter;
- EXPECT_EQ((uint64_t)16, (uint64_t)iter.key());
- EXPECT_EQ(A(1, 2, 3), iter.value());
- --iter;
- EXPECT_EQ((uint64_t)14, (uint64_t)iter.key());
- EXPECT_EQ(A(42, 0, 0), iter.value());
- ++iter;
- EXPECT_EQ((uint64_t)16, (uint64_t)iter.key());
- EXPECT_EQ(A(1, 2, 3), iter.value());
- --iter;
- --iter;
- EXPECT_EQ((uint64_t)11,(uint64_t) iter.key());
- EXPECT_EQ(A(4, 6, 0), iter.value());
- ++iter;
- ++iter;
- ++iter;
- EXPECT_EQ(multiMap.end(), iter);
- --iter;
- EXPECT_EQ((uint64_t)16, (uint64_t)iter.key());
- EXPECT_EQ(A(1, 2, 3), iter.value());
- --iter;
- EXPECT_EQ((uint64_t)14, (uint64_t)iter.key());
- EXPECT_EQ(A(42, 0, 0), iter.value());
- --iter;
- EXPECT_EQ((uint64_t)11,(uint64_t) iter.key());
- EXPECT_EQ(A(4, 6, 0), iter.value());
-}
-
-} // storage
-
diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp
index 1ea10003de1..f5e68dd3de1 100644
--- a/storage/src/tests/bucketdb/lockablemaptest.cpp
+++ b/storage/src/tests/bucketdb/lockablemaptest.cpp
@@ -1,9 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <vespa/vespalib/util/document_runnable.h>
-#include <vespa/storage/bucketdb/judymultimap.h>
-#include <vespa/storage/bucketdb/judymultimap.hpp>
-#include <vespa/storage/bucketdb/lockablemap.hpp>
#include <vespa/storage/bucketdb/btree_lockable_map.hpp>
#include <vespa/vespalib/gtest/gtest.h>
#include <gmock/gmock.h>
@@ -58,7 +55,8 @@ struct LockableMapTest : ::testing::Test {
using Map = MapT;
};
-using MapTypes = ::testing::Types<LockableMap<JudyMultiMap<A>>, bucketdb::BTreeLockableMap<A>>;
+// TODO add striped B-tree DB to this type set once ready
+using MapTypes = ::testing::Types<bucketdb::BTreeLockableMap<A>>;
VESPA_GTEST_TYPED_TEST_SUITE(LockableMapTest, MapTypes);
// Disable warnings emitted by gtest generated files when using typed tests
diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt
index 5f491d80697..23974cf2522 100644
--- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt
+++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt
@@ -9,8 +9,6 @@ vespa_add_library(storage_bucketdb OBJECT
bucketmanager.cpp
bucketmanagermetrics.cpp
generic_btree_bucket_database.cpp
- judyarray.cpp
- lockablemap.cpp
storbucketdb.cpp
DEPENDS
)
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index 392c2da871a..7b934af4bdd 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -2,7 +2,6 @@
#include "bucketmanager.h"
#include "minimumusedbitstracker.h"
-#include "lockablemap.hpp"
#include <iomanip>
#include <vespa/storage/common/content_bucket_space_repo.h>
#include <vespa/storage/common/nodestateupdater.h>
@@ -22,6 +21,7 @@
#include <vespa/config/config.h>
#include <vespa/config/helper/configgetter.hpp>
#include <chrono>
+#include <thread>
#include <vespa/log/bufferedlogger.h>
LOG_SETUP(".storage.bucketdb.manager");
diff --git a/storage/src/vespa/storage/bucketdb/judyarray.cpp b/storage/src/vespa/storage/bucketdb/judyarray.cpp
deleted file mode 100644
index cd348300d28..00000000000
--- a/storage/src/vespa/storage/bucketdb/judyarray.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "judyarray.h"
-#include <vespa/vespalib/util/exceptions.h>
-#include <iostream>
-#include <sstream>
-
-namespace storage {
-
-JudyArray::~JudyArray()
-{
- clear();
-}
-
-JudyArray::iterator
-JudyArray::find(key_type key, bool insertIfNonExisting, bool& preExisted)
-{
- Iterator iter(*this, key);
- if (insertIfNonExisting && (iter.end() || iter.key() != key)) {
- preExisted = false;
- insert(key, 0);
- iter = Iterator(*this, key);
- assert(iter.key() == key);
- } else if (iter.key() != key) {
- preExisted = false;
- iter = Iterator(*this);
- } else {
- preExisted = true;
- }
- return iter;
-}
-
-void
-JudyArray::Iterator::setValue(data_type val)
-{
- if (_data == 0) {
- throw vespalib::IllegalArgumentException(
- "Cannot set value of end() iterator", VESPA_STRLOC);
- }
- *_data = val;
-}
-
-void
-JudyArray::Iterator::remove()
-{
- if (_data == 0) {
- throw vespalib::IllegalArgumentException(
- "Cannot erase end() iterator", VESPA_STRLOC);
- }
- _parent->erase(_key);
-}
-
-bool
-JudyArray::operator==(const JudyArray& array) const
-{
- if (size() != array.size()) return false;
- for (JudyArray::const_iterator it1 = begin(), it2 = array.begin();
- it1 != end(); ++it1, ++it2)
- {
- if (*it1 != *it2) return false;
- }
- return true;
-}
-
-bool
-JudyArray::operator<(const JudyArray& array) const
-{
- if (size() != array.size()) return (size() < array.size());
- for (JudyArray::const_iterator it1 = begin(), it2 = array.begin();
- it1 != end(); ++it1, ++it2)
- {
- if (*it1 != *it2) return (*it1 < *it2);
- }
- return false;
-}
-
-JudyArray::size_type
-JudyArray::erase(key_type key)
-{
- JError_t err;
- size_type result = JudyLDel(&_judyArray, key, &err);
- if (result == 0 || result == 1) {
- return result;
- }
- std::ostringstream ost;
- ost << "Judy error in erase(" << std::hex << key << "): " << err.je_Errno;
- std::cerr << ost.str() << "\n";
- assert(false);
- return 0;
-}
-
-
-JudyArray::size_type
-JudyArray::size() const
-{
- key_type lastIndex = 0;
- --lastIndex; // Get last index in size independent way
- return JudyLCount(_judyArray, 0, lastIndex, PJE0);
-}
-
-void
-JudyArray::swap(JudyArray& other)
-{
- void* judyArray = _judyArray; // Save our variables
- _judyArray = other._judyArray; // Assign others to ours
- other._judyArray = judyArray; // Assign temporary to other
-}
-
-void
-JudyArray::print(std::ostream& out, bool, const std::string& indent) const
-{
- out << "JudyArray(";
- for (const_iterator i = begin(); i != end(); ++i) {
- out << "\n" << indent << " Key: " << i.key()
- << ", Value: " << i.value();
- }
- out << "\n" << indent << ")";
-}
-
-JudyArray::ConstIterator::ConstIterator(const JudyArray& arr)
- : _key(0), _data(0), _parent(const_cast<JudyArray*>(&arr)) {}
-
-JudyArray::ConstIterator::ConstIterator(const JudyArray& arr, key_type mykey)
- : _key(mykey), _data(0), _parent(const_cast<JudyArray*>(&arr))
-{
- _data = reinterpret_cast<data_type*>(
- JudyLFirst(_parent->_judyArray, &_key, PJE0));
-}
-
-void
-JudyArray::ConstIterator::print(std::ostream& out, bool, const std::string&) const
-{
- if (dynamic_cast<const Iterator*>(this) == 0) {
- out << "Const";
- }
- out << "Iterator(Key: " << _key << ", Valp: " << _data;
- if (_data) out << ", Val: " << *_data;
- out << ")";
-}
-
-JudyArray::Iterator::Iterator(JudyArray& arr)
- : ConstIterator(arr) {}
-
-JudyArray::Iterator::Iterator(JudyArray& arr, key_type mykey)
- : ConstIterator(arr, mykey) {}
-
-} // storage
diff --git a/storage/src/vespa/storage/bucketdb/judyarray.h b/storage/src/vespa/storage/bucketdb/judyarray.h
deleted file mode 100644
index 71db95d42c0..00000000000
--- a/storage/src/vespa/storage/bucketdb/judyarray.h
+++ /dev/null
@@ -1,211 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * @class JudyArray
- *
- * Implements a pair associative container on top of a judy array.
- *
- * NB: All iterators are invalidated after writing to judy array.
- *
- * NB: Using JudyArray's insert, one can only detect if the element already
- * existed, if the element didn't have the value 0. Since we don't want to
- * say that values cannot be 0, size is not counted outside of judy array, but
- * rather counts elements in the judy array when asked.
- *
- * @author Haakon Humberset
- */
-
-#pragma once
-
-#include <vespa/vespalib/util/printable.h>
-#include <cinttypes>
-#include <Judy.h>
-#include <cassert>
-
-namespace storage {
-
-class JudyArray : public vespalib::Printable
-{
- JudyArray(const JudyArray&); // Deny copying
- JudyArray& operator=(const JudyArray&);
-
-public:
- class Iterator;
- class ConstIterator;
-
- typedef Iterator iterator;
- typedef ConstIterator const_iterator;
- typedef unsigned long key_type;
- typedef unsigned long data_type;
- typedef std::pair<const key_type, data_type> value_type;
- typedef size_t size_type;
- typedef value_type& reference;
- typedef const value_type& const_reference;
- typedef value_type* pointer;
- typedef int difference_type;
-
- JudyArray() : _judyArray(NULL) {}
- virtual ~JudyArray();
-
- bool operator==(const JudyArray& array) const;
- bool operator!=(const JudyArray& array) const {
- return ! (*this == array);
- }
- bool operator<(const JudyArray& array) const;
-
- /** Warning: Size may be a O(n) function (Unknown implementation in judy) */
- size_type size() const;
- bool empty() const { return (begin() == end()); }
-
- iterator begin() { return Iterator(*this, 0); }
- iterator end() { return Iterator(*this); }
- const_iterator begin() const { return ConstIterator(*this, 0); }
- const_iterator end() const { return ConstIterator(*this); }
-
- void swap(JudyArray&);
-
- const_iterator find(key_type key) const;
- /**
- * Get iterator to value with given key. If non-existing, returns end(),
- * unless insert is true, in which case the element will be created.
- */
- iterator find(key_type key, bool insert, bool& preExisted);
- iterator find(key_type key) { bool b; return find(key, false, b); }
-
- const_iterator lower_bound(key_type key) const
- { return ConstIterator(*this, key); }
- iterator lower_bound(key_type key) { return Iterator(*this, key); }
-
- size_type erase(key_type key);
- void erase(iterator& iter) { iter.remove(); }
-
- void insert(key_type key, data_type val);
- void clear();
-
- data_type& operator[](key_type key);
- size_type getMemoryUsage() const;
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- class ConstIterator : public vespalib::Printable
- {
- public:
- ConstIterator& operator--();
- ConstIterator& operator++();
-
- bool operator==(const ConstIterator &cp) const;
- bool operator!=(const ConstIterator &cp) const {
- return ! (*this == cp);
- }
- value_type operator*() const { return value_type(_key, *_data); }
-
- bool end() const { return (_data == 0); }
- key_type key() const { return _key; }
- data_type value() const { return *_data; }
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
- protected:
- // For creating end() iterator
- ConstIterator(const JudyArray&);
- // Create iterator pointing to first element >= key.
- ConstIterator(const JudyArray&, key_type);
-
- key_type _key; // Key iterator currently points to
- data_type* _data; // Pointer to member pointed to, or 0 if end().
- JudyArray* _parent;
- friend class JudyArray;
- };
-
- class Iterator : public ConstIterator
- {
- public:
- Iterator& operator--()
- { return static_cast<Iterator&>(ConstIterator::operator--()); }
-
- Iterator& operator++()
- { return static_cast<Iterator&>(ConstIterator::operator++()); }
-
- void setValue(data_type val);
- void remove();
-
- private:
- Iterator(JudyArray&);
- Iterator(JudyArray&, key_type key);
- friend class JudyArray;
- };
-
-private:
- void *_judyArray;
- friend class Iterator;
- friend class ConstIterator;
-};
-
-inline JudyArray::const_iterator
-JudyArray::find(key_type key) const
-{
- ConstIterator iter(*this, key);
- if (!iter.end() && iter.key() != key) {
- iter = ConstIterator(*this);
- }
- return iter;
-}
-
-inline void
-JudyArray::insert(key_type key, data_type val)
-{
- data_type* valp = reinterpret_cast<data_type*>(
- JudyLIns(&_judyArray, key, PJE0));
- *valp = val;
-}
-
-inline void
-JudyArray::clear()
-{
- JudyLFreeArray(&_judyArray, PJE0);
-}
-
-inline JudyArray::data_type&
-JudyArray::operator[](key_type key)
-{
- data_type* valp = reinterpret_cast<data_type*>(
- JudyLGet(_judyArray, key, PJE0));
- if (valp == 0) {
- valp = reinterpret_cast<data_type*>(JudyLIns(&_judyArray, key, PJE0));
- *valp = 0;
- }
- return *valp;
-}
-
-inline JudyArray::size_type
-JudyArray::getMemoryUsage() const
-{
- return JudyLMemUsed(_judyArray);
-}
-
-inline JudyArray::ConstIterator&
-JudyArray::ConstIterator::operator--() // Prefix
-{
- if (!_data) {
- _data = reinterpret_cast<data_type*>(
- JudyLLast(_parent->_judyArray, &_key, PJE0));
- } else {
- _data = reinterpret_cast<data_type*>(
- JudyLPrev(_parent->_judyArray, &_key, PJE0));
- }
- return *this;
-}
-
-inline JudyArray::ConstIterator&
-JudyArray::ConstIterator::operator++() // Prefix
-{
- _data = reinterpret_cast<data_type*>(
- JudyLNext(_parent->_judyArray, &_key, PJE0));
- return *this;
-}
-
-inline bool
-JudyArray::ConstIterator::operator==(const JudyArray::ConstIterator &cp) const
-{
- return (_data == cp._data);
-}
-
-} // storage
diff --git a/storage/src/vespa/storage/bucketdb/judymultimap.h b/storage/src/vespa/storage/bucketdb/judymultimap.h
deleted file mode 100644
index b321f6a4dea..00000000000
--- a/storage/src/vespa/storage/bucketdb/judymultimap.h
+++ /dev/null
@@ -1,179 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * @class JudyMultiMap
- *
- * Layer on top of JudyArray, to create a map from the judy array key type,
- * to any of a given set of array types.
- *
- * The value arrays in here all starts with an unused object at index 0.
- * This is because 0 is used as unset value in judyarray, such that we can
- * easily detect if we replace or insert new entry.
- *
- * NB: The order of the template parameters type must be ordered such that
- * the types can include less and less.
- *
- * NB: All iterators are invalidated after writing to judy map.
- *
- * NB: Using JudyArray's insert, one can only detect if the element already
- * existed, if the element didn't have the value 0. Since we don't want to
- * say that values cannot be 0, size is not counted outside of judy array, but
- * rather counts elements in the judy array when asked.
- *
- * @author Haakon Humberset<
- */
-
-
-#pragma once
-
-#include <vespa/storage/bucketdb/judyarray.h>
-#include <vespa/vespalib/util/array.h>
-#include <vector>
-
-namespace storage {
-
-template<class Type0,
- class Type1 = Type0,
- class Type2 = Type1,
- class Type3 = Type2 >
-class JudyMultiMap : public vespalib::Printable {
-public:
- JudyMultiMap();
- ~JudyMultiMap();
-
- class Iterator;
- class ConstIterator;
- class ValueType;
-
- typedef Iterator iterator;
- typedef ConstIterator const_iterator;
- typedef JudyArray::key_type key_type;
- typedef Type3 mapped_type;
- typedef std::pair<const key_type, mapped_type> value_type;
- typedef JudyArray::size_type size_type;
-
- bool operator==(const JudyMultiMap& array) const;
- bool operator<(const JudyMultiMap& array) const;
-
- /** Warning: Size may be a O(n) function (Unknown implementation in judy) */
- size_type size() const;
- bool empty() const { return (begin() == end()); }
-
- iterator begin() { return Iterator(*this, 0); }
- iterator end() { return Iterator(*this); }
- const_iterator begin() const { return ConstIterator(*this, 0); }
- const_iterator end() const { return ConstIterator(*this); }
-
- void swap(JudyMultiMap&);
-
- const_iterator find(key_type key) const;
- /**
- * Get iterator to value with given key. If non-existing, returns end(),
- * unless insert is true, in which case the element will be created.
- */
- iterator find(key_type key, bool insert, bool& preExisted);
- iterator find(key_type key) { bool b; return find(key, false, b); }
-
- const_iterator lower_bound(key_type key) const
- { return ConstIterator(*this, key); }
- iterator lower_bound(key_type key) { return Iterator(*this, key); }
-
- size_type erase(key_type key);
-#if 0
- void erase(iterator& iter) { iter.remove(); }
-#endif
-
- void insert(key_type key, const Type3& val, bool& preExisted)
- {
- JudyArray::iterator it(_judyArray.find(key, true, preExisted));
- insert(it, val);
- }
- void clear();
-
- const mapped_type operator[](key_type key);
- size_type getMemoryUsage() const;
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- class ConstIterator : public vespalib::Printable
- {
- public:
- ConstIterator& operator--() { --_iterator; return *this; }
- ConstIterator& operator++() { ++_iterator; return *this; }
-
- bool operator==(const ConstIterator &cp) const;
- bool operator!=(const ConstIterator &cp) const {
- return ! (*this == cp);
- }
- value_type operator*() const;
-
- bool end() const { return _iterator.end(); }
- key_type key() const { return _iterator.key(); }
- mapped_type value() const;
-
- const std::pair<key_type, mapped_type>* operator->() const {
- _pair = std::pair<key_type, mapped_type>(_iterator.key(), value());
- return &_pair;
- }
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- protected:
- // For creating end() iterator
- ConstIterator(const JudyMultiMap&);
- // Create iterator pointing to first element >= key.
- ConstIterator(const JudyMultiMap&, key_type);
-
- JudyArray::ConstIterator _iterator;
- JudyMultiMap* _parent;
- friend class JudyMultiMap;
- mutable std::pair<key_type, mapped_type> _pair;
- };
-
- class Iterator : public ConstIterator
- {
- public:
- Iterator& operator--()
- { return static_cast<Iterator&>(ConstIterator::operator--()); }
-
- Iterator& operator++()
- { return static_cast<Iterator&>(ConstIterator::operator++()); }
-
- void setValue(const Type3& val);
- void remove();
-
- private:
- Iterator(JudyMultiMap&);
- Iterator(JudyMultiMap&, key_type key);
- friend class JudyMultiMap;
- };
-
-private:
- JudyArray _judyArray;
- typedef vespalib::Array<Type0> Type0Vector;
- typedef vespalib::Array<Type1> Type1Vector;
- typedef vespalib::Array<Type2> Type2Vector;
- typedef vespalib::Array<Type3> Type3Vector;
- Type0Vector _values0;
- Type1Vector _values1;
- Type2Vector _values2;
- Type3Vector _values3;
- std::vector<std::vector<typename Type0Vector::size_type> > _free;
- friend class Iterator;
- friend class ConstIterator;
-
- static int getType(JudyArray::data_type index) {
- return index >> (8 * sizeof(JudyArray::data_type) - 2);
- }
- static JudyArray::data_type getIndex(JudyArray::data_type index) {
- return ((index << 2) >> 2);
- }
- static JudyArray::data_type getValue(JudyArray::data_type type,
- JudyArray::data_type index)
- {
- return (type << (8 * sizeof(JudyArray::data_type) - 2) | index);
- }
- void insert(JudyArray::iterator& it, const Type3& val);
-};
-
-} // storage
-
diff --git a/storage/src/vespa/storage/bucketdb/judymultimap.hpp b/storage/src/vespa/storage/bucketdb/judymultimap.hpp
deleted file mode 100644
index 0d80bcfb28e..00000000000
--- a/storage/src/vespa/storage/bucketdb/judymultimap.hpp
+++ /dev/null
@@ -1,406 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "judymultimap.h"
-#include <vespa/vespalib/util/hdr_abort.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <vespa/vespalib/util/array.hpp>
-#include <set>
-#include <ostream>
-
-namespace storage {
-
-template<class T0, class T1, class T2, class T3>
-JudyMultiMap<T0, T1, T2, T3>::JudyMultiMap()
- : _values0(1), _values1(1), _values2(1), _values3(1), _free(4)
-{ }
-
-template<class T0, class T1, class T2, class T3>
-JudyMultiMap<T0, T1, T2, T3>::~JudyMultiMap() { }
-
-template<class T0, class T1, class T2, class T3>
-bool
-JudyMultiMap<T0, T1, T2, T3>::
-operator==(const JudyMultiMap<T0, T1, T2, T3>& map) const
-{
- if (size() != map.size()) return false;
- for (typename JudyMultiMap<T0, T1, T2, T3>::const_iterator
- it1 = begin(), it2 = map.begin(); it1 != end(); ++it1, ++it2)
- {
- assert(it2 != end());
- if (*it1 != *it2) return false;
- }
- return true;
-}
-
-template<class T0, class T1, class T2, class T3>
-bool
-JudyMultiMap<T0, T1, T2, T3>::
-operator<(const JudyMultiMap<T0, T1, T2, T3>& map) const
-{
- if (size() != map.size()) return (size() < map.size());
- for (typename JudyMultiMap<T0, T1, T2, T3>::const_iterator
- it1 = begin(), it2 = map.begin(); it1 != end(); ++it1, ++it2)
- {
- if (it1.key() != it2.key()) return (it1.key() < it2.key());
- if (it1.value() != it2.value()) return (it1.value() < it2.value());
- }
- return false;
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::size_type
-JudyMultiMap<T0, T1, T2, T3>::size() const
-{
- // First elements in all vectors is bogus, because we use value 0
- // to mean unset in judyarray. (To be able to detect if we overwrite)
- return _values0.size() + _values1.size()
- + _values2.size() + _values3.size() - 4
- - _free[0].size() - _free[1].size()
- - _free[2].size() - _free[3].size();
-}
-
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::
-swap(JudyMultiMap<T0, T1, T2, T3>& other)
-{
- _judyArray.swap(other._judyArray);
- _values0.swap(other._values0);
- _values1.swap(other._values1);
- _values2.swap(other._values2);
- _values3.swap(other._values3);
- _free.swap(other._free);
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::const_iterator
-JudyMultiMap<T0, T1, T2, T3>::find(key_type key) const
-{
- ConstIterator iter(*this, key);
- if (!iter.end() && iter.key() != key) {
- iter = ConstIterator(*this);
- }
- return iter;
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::iterator
-JudyMultiMap<T0, T1, T2, T3>::find(key_type key, bool insertIfNonExisting,
- bool& preExisted)
-{
- Iterator iter(*this, key);
- if (insertIfNonExisting && (iter.end() || iter.key() != key)) {
- insert(key, T3(), preExisted);
- iter = Iterator(*this, key);
- assert(iter.key() == key);
- } else if (iter.key() != key) {
- preExisted = false;
- iter = Iterator(*this);
- } else {
- preExisted = true;
- }
- return iter;
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::size_type
-JudyMultiMap<T0, T1, T2, T3>::erase(key_type key)
-{
- JudyArray::iterator it = _judyArray.find(key);
- if (it == _judyArray.end()) return 0;
- _free[getType(it.value())].push_back(getIndex(it.value()));
- _judyArray.erase(key);
- return 1;
-}
-
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::clear()
-{
- _judyArray.clear();
- _values0.resize(1);
- _values1.resize(1);
- _values2.resize(1);
- _values3.resize(1);
- _free[0].clear();
- _free[1].clear();
- _free[2].clear();
- _free[3].clear();
-}
-
-template<class T0, class T1, class T2, class T3>
-const typename JudyMultiMap<T0, T1, T2, T3>::mapped_type
-JudyMultiMap<T0, T1, T2, T3>::operator[](key_type key)
-{
- bool preExisted;
- JudyArray::iterator it = _judyArray.find(key, true, preExisted);
- // If it doesn't already exist, insert
- if (it.value() == 0) {
- if (_free[0].empty()) {
- it.setValue(getValue(0, _values0.size()));
- _values0.push_back(T0());
- } else {
- it.setValue(getValue(0, _free[0].back()));
- _values0[_free[0].back()] = T0();
- _free[0].pop_back();
- }
- }
- switch (getType(it.value())) {
- case 0: return _values0[getIndex(it.value())];
- case 1: return _values1[getIndex(it.value())];
- case 2: return _values2[getIndex(it.value())];
- case 3: return _values3[getIndex(it.value())];
- default: HDR_ABORT("should not be reached");
- }
- return T0(); // Avoid warning of no return
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::size_type
-JudyMultiMap<T0, T1, T2, T3>::getMemoryUsage() const
-{
- return _judyArray.getMemoryUsage()
- + sizeof(T0) * _values0.capacity()
- + sizeof(T1) * _values1.capacity()
- + sizeof(T2) * _values2.capacity()
- + sizeof(T3) * _values3.capacity()
- + sizeof(typename Type0Vector::size_type)
- * (_free[0].capacity() + _free[1].capacity() +
- _free[2].capacity() + _free[3].capacity());
-}
-
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::
-print(std::ostream& out, bool verbose, const std::string& indent) const
-{
- out << "JudyMultiMap(";
-
- if (verbose) {
- for (const_iterator i = begin(); i != end(); ++i) {
- out << "\n" << indent << " ";
- i.print(out, verbose, indent + " ");
- }
- }
-
- if (_values0.size() > 1) {
- std::set<typename Type0Vector::size_type> free(
- _free[0].begin(), _free[0].end());
- assert(free.size() == _free[0].size());
- out << "\n" << indent << " Type0 " << (_values0.size()-1)
- << " entries, " << free.size() << " free {";
-
- if (verbose) {
- for (uint32_t i=1; i<_values0.size(); ++i) {
- out << "\n" << indent << " ";
- if (free.find(i) != free.end()) { out << "free"; }
- else { out << _values0[i]; }
- }
- }
- out << "\n" << indent << " }";
- }
- if (_values1.size() > 1) {
- std::set<typename Type0Vector::size_type> free(
- _free[1].begin(), _free[1].end());
- assert(free.size() == _free[1].size());
- out << "\n" << indent << " Type1 " << (_values1.size()-1)
- << " entries, " << free.size() << " free {";
- if (verbose) {
- for (uint32_t i=1; i<_values1.size(); ++i) {
- out << "\n" << indent << " ";
- if (free.find(i) != free.end()) { out << "free"; }
- else { out << _values1[i]; }
- }
- }
- out << "\n" << indent << " }";
- }
- if (_values2.size() > 1) {
- std::set<typename Type0Vector::size_type> free(
- _free[2].begin(), _free[2].end());
- assert(free.size() == _free[2].size());
- out << "\n" << indent << " Type2 " << (_values2.size()-1)
- << " entries, " << free.size() << " free {";
- if (verbose) {
- for (uint32_t i=1; i<_values2.size(); ++i) {
- out << "\n" << indent << " ";
- if (free.find(i) != free.end()) { out << "free"; }
- else { out << _values2[i]; }
- }
- }
- out << "\n" << indent << " }";
- }
-
- if (_values3.size() > 1) {
- std::set<typename Type0Vector::size_type> free(
- _free[3].begin(), _free[3].end());
- assert(free.size() == _free[3].size());
- out << "\n" << indent << " Type3 " << (_values3.size()-1)
- << " entries, " << free.size() << " free {";
-
- if (verbose) {
- for (uint32_t i=1; i<_values3.size(); ++i) {
- out << "\n" << indent << " ";
- if (free.find(i) != free.end()) { out << "free"; }
- else { out << _values3[i]; }
- }
- }
- out << "\n" << indent << " }";
- }
- if (!empty()) { out << "\n" << indent; }
- out << ")";
-}
-
-template<class T0, class T1, class T2, class T3>
-JudyMultiMap<T0, T1, T2, T3>::
-ConstIterator::ConstIterator(const JudyMultiMap<T0, T1, T2, T3>& map)
- : _iterator(map._judyArray.end()),
- _parent(const_cast<JudyMultiMap<T0, T1, T2, T3>*>(&map))
-{
-}
-
-template<class T0, class T1, class T2, class T3>
-JudyMultiMap<T0, T1, T2, T3>::
-ConstIterator::ConstIterator(const JudyMultiMap<T0, T1, T2, T3>& map,
- key_type mykey)
- : _iterator(map._judyArray.lower_bound(mykey)),
- _parent(const_cast<JudyMultiMap<T0, T1, T2, T3>*>(&map))
-{
-}
-
-template<class T0, class T1, class T2, class T3>
-bool
-JudyMultiMap<T0, T1, T2, T3>::
-ConstIterator::operator==(const JudyMultiMap::ConstIterator &cp) const
-{
- return (_iterator == cp._iterator);
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::value_type
-JudyMultiMap<T0, T1, T2, T3>::ConstIterator::operator*() const
-{
- switch (getType(_iterator.value())) {
- case 0: return value_type(
- _iterator.key(), _parent->_values0[getIndex(_iterator.value())]);
- case 1: return value_type(
- _iterator.key(), _parent->_values1[getIndex(_iterator.value())]);
- case 2: return value_type(
- _iterator.key(), _parent->_values2[getIndex(_iterator.value())]);
- case 3: return value_type(
- _iterator.key(), _parent->_values3[getIndex(_iterator.value())]);
- default:
- HDR_ABORT("should not be reached");
- }
-}
-
-template<class T0, class T1, class T2, class T3>
-typename JudyMultiMap<T0, T1, T2, T3>::mapped_type
-JudyMultiMap<T0, T1, T2, T3>::ConstIterator::value() const
-{
- switch (getType(_iterator.value())) {
- case 0: return _parent->_values0[getIndex(_iterator.value())];
- case 1: return _parent->_values1[getIndex(_iterator.value())];
- case 2: return _parent->_values2[getIndex(_iterator.value())];
- case 3: return _parent->_values3[getIndex(_iterator.value())];
- default:
- HDR_ABORT("should not be reached");
- }
-}
-
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::
-ConstIterator::print(std::ostream& out, bool, const std::string&) const
-{
- if (dynamic_cast<const Iterator*>(this) == 0) {
- out << "Const";
- }
- out << "Iterator(Key: " << _iterator.key() << ", Value: " << value() << ")";
-}
-
-template<class T0, class T1, class T2, class T3>
-JudyMultiMap<T0, T1, T2, T3>::
-Iterator::Iterator(JudyMultiMap<T0, T1, T2, T3>& map)
- : ConstIterator(map) {}
-
-template<class T0, class T1, class T2, class T3>
-JudyMultiMap<T0, T1, T2, T3>::
-Iterator::Iterator(JudyMultiMap<T0, T1, T2, T3>& map, key_type mykey)
- : ConstIterator(map, mykey) {}
-
-#if 0
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::Iterator::setValue(const T3& val)
-{
- if (this->_iterator.end()) {
- throw vespalib::IllegalArgumentException(
- "Cannot set value of end() iterator", VESPA_STRLOC);
- }
- insert(this->iterator, val);
-}
-
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::Iterator::remove()
-{
- if (this->_iterator.end()) {
- throw vespalib::IllegalArgumentException(
- "Cannot erase end() iterator", VESPA_STRLOC);
- }
- int type = getType(this->_iterator.value());
- _free[type].push_back(getIndex(this->_iterator.value()));
- this->_iterator.remove();
-}
-
-#endif
-
-template<class T0, class T1, class T2, class T3>
-void
-JudyMultiMap<T0, T1, T2, T3>::insert(JudyArray::iterator& it, const T3& val)
-{
- // Find the type we need to save 'val' as
- int type;
- if (T0::mayContain(val)) { type = 0; }
- else if (T1::mayContain(val)) { type = 1; }
- else if (T2::mayContain(val)) { type = 2; }
- else { type = 3; }
- // If already pointing to some value, free that resource.
- int oldtype = getType(it.value());
- int index = getIndex(it.value());
- if (index != 0) {
- _free[oldtype].push_back(index);
- }
- // Insert value into new spot
- if (_free[type].empty()) {
- switch (type) {
- case 0: it.setValue(getValue(type, _values0.size()));
- _values0.push_back(val);
- break;
- case 1: it.setValue(getValue(type, _values1.size()));
- _values1.push_back(T1(val));
- break;
- case 2: it.setValue(getValue(type, _values2.size()));
- _values2.push_back(T2(val));
- break;
- case 3: it.setValue(getValue(type, _values3.size()));
- _values3.push_back(T3(val));
- break;
- default: assert(false);
- }
- } else {
- it.setValue(getValue(type, _free[type].back()));
- switch (type) {
- case 0: _values0[_free[type].back()] = val; break;
- case 1: _values1[_free[type].back()] = val; break;
- case 2: _values2[_free[type].back()] = val; break;
- case 3: _values3[_free[type].back()] = val; break;
- default: assert(false);
- }
- _free[type].pop_back();
- }
-}
-
-} // storage
-
diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.cpp b/storage/src/vespa/storage/bucketdb/lockablemap.cpp
deleted file mode 100644
index cb5cbfbc203..00000000000
--- a/storage/src/vespa/storage/bucketdb/lockablemap.cpp
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "lockablemap.hpp"
-#include "storagebucketinfo.h"
-#include "judymultimap.h"
-
-namespace storage {
-
-uint8_t
-getMinDiffBits(uint16_t minBits, const document::BucketId& a, const document::BucketId& b) {
- for (uint32_t i = minBits; i <= std::min(a.getUsedBits(), b.getUsedBits()); i++) {
- document::BucketId a1(i, a.getRawId());
- document::BucketId b1(i, b.getRawId());
- if (b1.getId() != a1.getId()) {
- return i;
- }
- }
- return minBits;
-}
-
-bool
-checkContains(document::BucketId::Type key, const document::BucketId& bucket,
- document::BucketId& result, document::BucketId::Type& keyResult)
-{
- document::BucketId id = document::BucketId(document::BucketId::keyToBucketId(key));
- if (id.contains(bucket)) {
- result = id;
- keyResult = key;
- return true;
- }
-
- return false;
-}
-
-using bucketdb::StorageBucketInfo;
-
-template class LockableMap<storage::JudyMultiMap<StorageBucketInfo, StorageBucketInfo, StorageBucketInfo, StorageBucketInfo> >;
-
-}
diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h
deleted file mode 100644
index 168439786e9..00000000000
--- a/storage/src/vespa/storage/bucketdb/lockablemap.h
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * A map wrapper, adding locking to the map entries. It provides the
- * following:
- *
- * - Guarantees thread safety.
- * - Each returned value is given within a wrapper. As long as the
- * wrapper for the value exist, this entry is locked in the map.
- * This does not prevent other values from being used. Wrappers can
- * be copied. Reference counting ensures value is locked until last
- * wrapper copy dies.
- * - Built in function for iterating taking a functor. Halts when
- * encountering locked values.
- */
-#pragma once
-
-#include "abstract_bucket_map.h"
-#include <map>
-#include <vespa/vespalib/util/printable.h>
-#include <vespa/vespalib/stllike/hash_map.h>
-#include <vespa/vespalib/stllike/hash_set.h>
-#include <vespa/document/bucket/bucketid.h>
-#include <vespa/vespalib/util/time.h>
-#include <mutex>
-#include <condition_variable>
-#include <cassert>
-
-namespace storage {
-
-template <typename Map>
-class LockableMap final
- : public bucketdb::AbstractBucketMap<typename Map::mapped_type>
-{
-public:
- using ParentType = bucketdb::AbstractBucketMap<typename Map::mapped_type>;
- using WrappedEntry = typename ParentType::WrappedEntry;
- using key_type = typename ParentType::key_type;
- using mapped_type = typename ParentType::mapped_type;
- using LockId = typename ParentType::LockId;
- using EntryMap = typename ParentType::EntryMap;
- using Decision = typename ParentType::Decision;
- using BucketId = document::BucketId;
-
- LockableMap();
- ~LockableMap();
- bool operator==(const LockableMap& other) const;
- bool operator!=(const LockableMap& other) const {
- return ! (*this == other);
- }
- bool operator<(const LockableMap& other) const;
- size_t size() const noexcept override;
- size_t getMemoryUsage() const noexcept override;
- vespalib::MemoryUsage detailed_memory_usage() const noexcept override;
- bool empty() const noexcept override;
- void swap(LockableMap&);
-
- WrappedEntry get(const key_type& key, const char* clientId, bool createIfNonExisting) override;
- WrappedEntry get(const key_type& key, const char* clientId) {
- return get(key, clientId, false);
- }
-
- bool erase(const key_type& key, const char* clientId, bool haslock) override;
- void insert(const key_type& key, const mapped_type& value,
- const char* clientId, bool haslock, bool& preExisted) override;
-
- bool erase(const key_type& key, const char* clientId)
- { return erase(key, clientId, false); }
- void insert(const key_type& key, const mapped_type& value,
- const char* clientId, bool& preExisted)
- { return insert(key, value, clientId, false, preExisted); }
- void clear();
-
- void print(std::ostream& out, bool verbose, const std::string& indent) const override;
-
- /**
- * Returns all buckets in the bucket database that can contain the given
- * bucket. Usually, there should be only one such bucket, but in the case
- * of inconsistent splitting, there may be more than one.
- */
- EntryMap getContained(const BucketId& bucketId, const char* clientId) override;
-
- /**
- * Returns all buckets in the bucket database that can contain the given
- * bucket, and all buckets that that bucket contains.
- */
- EntryMap getAll(const BucketId& bucketId, const char* clientId) override;
-
- /**
- * Returns true iff bucket has no superbuckets or sub-buckets in the
- * database. Usage assumption is that any operation that can cause the
- * bucket to become inconsistent will require taking its lock, so by
- * requiring the lock to be provided here we avoid race conditions.
- */
- bool isConsistent(const WrappedEntry& entry) override;
-
- void showLockClients(vespalib::asciistream & out) const override;
-
-private:
- struct hasher {
- size_t operator () (const LockId & lid) const { return lid.hash(); }
- };
- class LockIdSet : public vespalib::hash_set<LockId, hasher> {
- typedef vespalib::hash_set<LockId, hasher> Hash;
- public:
- LockIdSet();
- ~LockIdSet();
- void print(std::ostream& out, bool verbose, const std::string& indent) const;
- bool exist(const LockId& lid) const { return this->find(lid) != Hash::end(); }
- size_t getMemoryUsage() const;
- };
-
- class LockWaiters {
- typedef vespalib::hash_map<size_t, LockId> WaiterMap;
- public:
- typedef size_t Key;
- typedef typename WaiterMap::const_iterator const_iterator;
- LockWaiters();
- ~LockWaiters();
- Key insert(const LockId & lid);
- void erase(Key id) { _map.erase(id); }
- const_iterator begin() const { return _map.begin(); }
- const_iterator end() const { return _map.end(); }
- private:
- Key _id;
- WaiterMap _map;
- };
-
- class ReadGuardImpl;
-
- Map _map;
- mutable std::mutex _lock;
- std::condition_variable _cond;
- LockIdSet _lockedKeys;
- LockWaiters _lockWaiters;
-
- void unlock(const key_type& key) override;
- bool findNextKey(key_type& key, mapped_type& val, const char* clientId,
- std::unique_lock<std::mutex> &guard);
- bool handleDecision(key_type& key, mapped_type& val, Decision decision);
- void acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard);
-
- void do_for_each_mutable_unordered(std::function<Decision(uint64_t, mapped_type&)> func,
- const char* clientId) override;
-
- void do_for_each(std::function<Decision(uint64_t, const mapped_type&)> func,
- const char* clientId,
- const key_type& first,
- const key_type& last) override;
-
- void do_for_each_chunked(std::function<Decision(uint64_t, const mapped_type&)> func,
- const char* clientId,
- vespalib::duration yieldTime,
- uint32_t chunkSize) override;
-
- std::unique_ptr<bucketdb::ReadGuard<typename Map::mapped_type>> do_acquire_read_guard() const override;
-
- /**
- * Process up to `chunkSize` bucket database entries from--and possibly
- * including--the bucket pointed to by `key`.
- *
- * Returns true if additional chunks may be processed after the call to
- * this function has returned, false if iteration has completed or if
- * `functor` returned an abort-decision.
- *
- * Modifies `key` in-place to point to the next key to process for the next
- * invocation of this function.
- */
- bool processNextChunk(std::function<Decision(uint64_t, const mapped_type&)>& func,
- key_type& key,
- const char* clientId,
- uint32_t chunkSize);
-
- /**
- * Returns the given bucket, its super buckets and its sub buckets.
- */
- void getAllWithoutLocking(const BucketId& bucket,
- std::vector<BucketId::Type>& keys);
-
- /**
- * Retrieves the most specific bucket id (highest used bits) that matches
- * the given bucket.
- *
- * If a match is found, result is set to the bucket id found, and keyResult
- * is set to the corresponding key (reversed)
- *
- * If not found, nextKey is set to the key after one that could have
- * matched and we return false.
- */
- bool getMostSpecificMatch(const BucketId& bucket,
- BucketId& result,
- BucketId::Type& keyResult,
- BucketId::Type& nextKey);
-
- /**
- * Finds all buckets that can contain the given bucket, except for the
- * bucket itself (that is, its super buckets)
- */
- void getAllContaining(const BucketId& bucket,
- std::vector<BucketId::Type>& keys);
-
- /**
- * Find the given list of keys in the map and add them to the map of
- * results, locking them in the process.
- */
- void addAndLockResults(const std::vector<BucketId::Type>& keys,
- const char* clientId,
- std::map<BucketId, WrappedEntry>& results,
- std::unique_lock<std::mutex> &guard);
-};
-
-} // storage
-
diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp
deleted file mode 100644
index f6ae420cb34..00000000000
--- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp
+++ /dev/null
@@ -1,665 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "lockablemap.h"
-#include <vespa/vespalib/util/hdr_abort.h>
-#include <vespa/vespalib/stllike/asciistream.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
-#include <vespa/vespalib/stllike/hash_set.hpp>
-#include <thread>
-#include <chrono>
-#include <ostream>
-
-namespace storage {
-
-template<typename Map>
-LockableMap<Map>::LockIdSet::LockIdSet() : Hash() { }
-
-template<typename Map>
-LockableMap<Map>::LockIdSet::~LockIdSet() = default;
-
-template<typename Map>
-size_t
-LockableMap<Map>::LockIdSet::getMemoryUsage() const {
- return Hash::getMemoryConsumption();
-}
-
-template<typename Map>
-LockableMap<Map>::LockWaiters::LockWaiters() : _id(0), _map() { }
-
-template<typename Map>
-LockableMap<Map>::LockWaiters::~LockWaiters() = default;
-
-template<typename Map>
-size_t
-LockableMap<Map>::LockWaiters::insert(const LockId & lid) {
- Key id(_id++);
- _map.insert(typename WaiterMap::value_type(id, lid));
- return id;
-}
-
-template<typename Map>
-LockableMap<Map>::LockableMap()
- : _map(),
- _lock(),
- _cond(),
- _lockedKeys(),
- _lockWaiters()
-{}
-
-template<typename Map>
-LockableMap<Map>::~LockableMap() = default;
-
-template<typename Map>
-bool
-LockableMap<Map>::operator==(const LockableMap<Map>& other) const
-{
- std::lock_guard<std::mutex> guard(_lock);
- std::lock_guard<std::mutex> guard2(other._lock);
- return (_map == other._map);
-}
-
-template<typename Map>
-bool
-LockableMap<Map>::operator<(const LockableMap<Map>& other) const
-{
- std::lock_guard<std::mutex> guard(_lock);
- std::lock_guard<std::mutex> guard2(other._lock);
- return (_map < other._map);
-}
-
-template<typename Map>
-size_t
-LockableMap<Map>::size() const noexcept
-{
- std::lock_guard<std::mutex> guard(_lock);
- return _map.size();
-}
-
-template<typename Map>
-size_t
-LockableMap<Map>::getMemoryUsage() const noexcept
-{
- std::lock_guard<std::mutex> guard(_lock);
- return _map.getMemoryUsage() + _lockedKeys.getMemoryUsage() +
- sizeof(std::mutex) + sizeof(std::condition_variable);
-}
-
-template <typename Map>
-vespalib::MemoryUsage LockableMap<Map>::detailed_memory_usage() const noexcept {
- // We don't have any details for this map type, just count everything as "allocated".
- size_t used = getMemoryUsage();
- vespalib::MemoryUsage mem_usage;
- mem_usage.incAllocatedBytes(used);
- mem_usage.incUsedBytes(used);
- return mem_usage;
-}
-
-template<typename Map>
-bool
-LockableMap<Map>::empty() const noexcept
-{
- std::lock_guard<std::mutex> guard(_lock);
- return _map.empty();
-}
-
-template<typename Map>
-void
-LockableMap<Map>::swap(LockableMap<Map>& other)
-{
- std::lock_guard<std::mutex> guard(_lock);
- std::lock_guard<std::mutex> guard2(other._lock);
- return _map.swap(other._map);
-}
-
-template<typename Map>
-void LockableMap<Map>::acquireKey(const LockId & lid, std::unique_lock<std::mutex> &guard)
-{
- if (_lockedKeys.exist(lid)) {
- typename LockWaiters::Key waitId(_lockWaiters.insert(lid));
- while (_lockedKeys.exist(lid)) {
- _cond.wait(guard);
- }
- _lockWaiters.erase(waitId);
- }
-}
-
-template<typename Map>
-typename LockableMap<Map>::WrappedEntry
-LockableMap<Map>::get(const key_type& key, const char* clientId, bool createIfNonExisting)
-{
- LockId lid(key, clientId);
- std::unique_lock<std::mutex> guard(_lock);
- acquireKey(lid, guard);
- bool preExisted = false;
- typename Map::iterator it =
- _map.find(key, createIfNonExisting, preExisted);
-
- if (it == _map.end()) {
- return WrappedEntry();
- }
- _lockedKeys.insert(lid);
- return WrappedEntry(*this, key, it->second, clientId, preExisted);
-}
-
-template<typename Map>
-bool
-LockableMap<Map>::erase(const key_type& key, const char* client_id, bool has_lock)
-{
- LockId lid(key, client_id);
- std::unique_lock<std::mutex> guard(_lock);
- if (!has_lock) {
- acquireKey(lid, guard);
- }
- return _map.erase(key);
-}
-
-template<typename Map>
-void
-LockableMap<Map>::insert(const key_type& key, const mapped_type& value,
- const char* client_id, bool has_lock, bool& preExisted)
-{
- LockId lid(key, client_id);
- std::unique_lock<std::mutex> guard(_lock);
- if (!has_lock) {
- acquireKey(lid, guard);
- }
- _map.insert(key, value, preExisted);
-}
-
-template<typename Map>
-void
-LockableMap<Map>::clear()
-{
- std::lock_guard<std::mutex> guard(_lock);
- _map.clear();
-}
-
-template<typename Map>
-bool
-LockableMap<Map>::findNextKey(key_type& key, mapped_type& val,
- const char* clientId,
- std::unique_lock<std::mutex> &guard)
-{
- // Wait for next value to unlock.
- typename Map::iterator it(_map.lower_bound(key));
- while (it != _map.end() && _lockedKeys.exist(LockId(it->first, ""))) {
- auto wait_id = _lockWaiters.insert(LockId(it->first, clientId));
- _cond.wait(guard);
- _lockWaiters.erase(wait_id);
- it = _map.lower_bound(key);
- }
- if (it == _map.end()) {
- return true;
- }
- key = it->first;
- val = it->second;
- return false;
-}
-
-template<typename Map>
-bool
-LockableMap<Map>::handleDecision(key_type& key, mapped_type& val,
- Decision decision)
-{
- bool b;
- switch (decision) {
- case Decision::UPDATE:
- _map.insert(key, val, b);
- break;
- case Decision::REMOVE:
- _map.erase(key);
- break;
- case Decision::ABORT:
- return true;
- case Decision::CONTINUE:
- break;
- default:
- HDR_ABORT("should not be reached");
- }
- return false;
-}
-
-template<typename Map>
-void LockableMap<Map>::do_for_each_mutable_unordered(std::function<Decision(uint64_t, mapped_type&)> func,
- const char* clientId)
-{
- key_type key = 0;
- mapped_type val;
- std::unique_lock<std::mutex> guard(_lock);
- while (true) {
- if (findNextKey(key, val, clientId, guard)) {
- return;
- }
- Decision d(func(const_cast<const key_type&>(key), val));
- if (handleDecision(key, val, d)) {
- return;
- }
- ++key;
- }
-}
-
-template<typename Map>
-void LockableMap<Map>::do_for_each(std::function<Decision(uint64_t, const mapped_type&)> func,
- const char* clientId,
- const key_type& first,
- const key_type& last)
-{
- key_type key = first;
- mapped_type val;
- std::unique_lock<std::mutex> guard(_lock);
- while (true) {
- if (findNextKey(key, val, clientId, guard) || key > last) {
- return;
- }
- Decision d(func(const_cast<const key_type&>(key), val));
- assert(d == Decision::ABORT || d == Decision::CONTINUE);
- if (handleDecision(key, val, d)) {
- return;
- }
- ++key;
- }
-}
-
-template <typename Map>
-bool
-LockableMap<Map>::processNextChunk(std::function<Decision(uint64_t, const mapped_type&)>& func,
- key_type& key,
- const char* clientId,
- const uint32_t chunkSize)
-{
- mapped_type val;
- std::unique_lock<std::mutex> guard(_lock);
- for (uint32_t processed = 0; processed < chunkSize; ++processed) {
- if (findNextKey(key, val, clientId, guard)) {
- return false;
- }
- Decision d(func(const_cast<const key_type&>(key), val));
- if (handleDecision(key, val, d)) {
- return false;
- }
- ++key;
- }
- return true;
-}
-
-template <typename Map>
-void LockableMap<Map>::do_for_each_chunked(std::function<Decision(uint64_t, const mapped_type&)> func,
- const char* clientId,
- vespalib::duration yieldTime,
- uint32_t chunkSize)
-{
- key_type key{};
- while (processNextChunk(func, key, clientId, chunkSize)) {
- // Rationale: delay iteration for as short a time as possible while
- // allowing another thread blocked on the main DB mutex to acquire it
- // in the meantime. Simply yielding the thread does not have the
- // intended effect with the Linux scheduler.
- // This is a pragmatic stop-gap solution; a more robust change requires
- // the redesign of bucket DB locking and signalling semantics in the
- // face of blocked point lookups.
- std::this_thread::sleep_for(yieldTime);
- }
-}
-
-// TODO This is a placeholder that has to work around the const-ness and type quirks of
-// the legacy LockableMap implementation. In particular, it offers no snapshot isolation
-// at all, nor does it support the "get parents and self" bucket lookup operation.
-template <typename Map>
-class LockableMap<Map>::ReadGuardImpl final
- : public bucketdb::ReadGuard<typename Map::mapped_type>
-{
- const LockableMap<Map>& _map;
-public:
- using mapped_type = typename Map::mapped_type;
-
- explicit ReadGuardImpl(const LockableMap<Map>& map) : _map(map) {}
- ~ReadGuardImpl() override = default;
-
- std::vector<mapped_type> find_parents_and_self(const document::BucketId&) const override {
- abort(); // Finding just parents+self isn't supported by underlying legacy DB API!
- }
-
- std::vector<mapped_type> find_parents_self_and_children(const document::BucketId& bucket) const override {
- auto& mutable_map = const_cast<LockableMap<Map>&>(_map); // _map is thread safe.
- auto locked_entries = mutable_map.getAll(bucket, "ReadGuardImpl::find_parents_self_and_children");
- std::vector<mapped_type> entries;
- entries.reserve(locked_entries.size());
- for (auto& e : locked_entries) {
- entries.emplace_back(*e.second);
- }
- return entries;
- }
-
- void for_each(std::function<void(uint64_t, const mapped_type&)> func) const override {
- auto decision_wrapper = [&func](uint64_t key, const mapped_type& value) -> Decision {
- func(key, value);
- return Decision::CONTINUE;
- };
- auto& mutable_map = const_cast<LockableMap<Map>&>(_map); // _map is thread safe.
- mutable_map.for_each_chunked(std::move(decision_wrapper), "ReadGuardImpl::for_each");
- }
-
- [[nodiscard]] uint64_t generation() const noexcept override {
- return 0;
- }
-};
-
-template <typename Map>
-std::unique_ptr<bucketdb::ReadGuard<typename Map::mapped_type>>
-LockableMap<Map>::do_acquire_read_guard() const {
- return std::make_unique<ReadGuardImpl>(*this);
-}
-
-template<typename Map>
-void
-LockableMap<Map>::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- std::lock_guard<std::mutex> guard(_lock);
- out << "LockableMap {\n" << indent << " ";
-
- if (verbose) {
- for (const auto entry : _map) {
- out << "Key: " << BucketId(BucketId::keyToBucketId(entry.first))
- << " Value: " << entry.second << "\n" << indent << " ";
- }
-
- out << "\n" << indent << " Locked keys: ";
- _lockedKeys.print(out, verbose, indent + " ");
- }
- out << "} : ";
-
- out << _map;
-}
-
-template<typename Map>
-void
-LockableMap<Map>::LockIdSet::print(std::ostream& out, bool verbose,
- const std::string& indent) const
-{
- out << "hash {";
- for (typename Hash::const_iterator it(Hash::begin()), mt(Hash::end()); it != mt; it++) {
- if (verbose) {
- out << "\n" << indent << " ";
- } else {
- out << " ";
- }
-
- out << *it;
- }
- if (verbose) out << "\n" << indent;
- out << " }";
-}
-
-template<typename Map>
-void
-LockableMap<Map>::unlock(const key_type& key)
-{
- std::lock_guard<std::mutex> guard(_lock);
- _lockedKeys.erase(LockId(key, ""));
- _cond.notify_all();
-}
-
-/**
- * Check whether the given key contains the given bucket.
- * Sets result to the bucket corresponding to the key, and keyResult
- * to the key if true.
- */
-bool
-checkContains(document::BucketId::Type key, const document::BucketId& bucket,
- document::BucketId& result, document::BucketId::Type& keyResult);
-
-/**
- * Retrieves the most specific bucket id (highest used bits) that contains
- * the given bucket.
- *
- * If a match is found, result is set to the bucket id found, and keyResult is
- * set to the corresponding key (reversed)
- *
- * If not found, nextKey is set to the key after one that could have matched
- * and we return false.
- */
-template<typename Map>
-bool
-LockableMap<Map>::getMostSpecificMatch(const BucketId& bucket,
- BucketId& result,
- BucketId::Type& keyResult,
- BucketId::Type& nextKey)
-{
- typename Map::const_iterator iter = _map.lower_bound(bucket.toKey());
-
- nextKey = 0;
-
- // We should now have either the bucket we are looking for
- // (if the exact bucket exists), or one right after.
- if (iter != _map.end()) {
- nextKey = iter->first;
-
- if (checkContains(iter->first, bucket, result, keyResult)) {
- return true;
- }
- }
-
- if (iter != _map.begin()) {
- --iter; // If iter was map.end(), we should now end up at the last item in the map
- nextKey = iter->first;
-
- if (checkContains(iter->first, bucket, result, keyResult)) {
- return true;
- }
- }
-
- return false;
-}
-
-/**
- * Finds all buckets that can contain the given bucket, except for the bucket
- * itself.
- */
-template<typename Map>
-void
-LockableMap<Map>::getAllContaining(const BucketId& bucket,
- std::vector<BucketId::Type>& keys)
-{
- BucketId id = bucket;
-
- // Find other buckets that contain this bucket.
- // TODO: Optimize?
- while (id.getUsedBits() > 1) {
- id.setUsedBits(id.getUsedBits() - 1);
- id = id.stripUnused();
- BucketId::Type key = id.toKey();
-
- typename Map::const_iterator iter = _map.find(key);
- if (iter != _map.end()) {
- keys.push_back(key);
- }
- }
-}
-
-template<typename Map>
-void
-LockableMap<Map>::addAndLockResults(
- const std::vector<BucketId::Type>& keys,
- const char* clientId,
- std::map<BucketId, WrappedEntry>& results,
- std::unique_lock<std::mutex> &guard)
-{
- // Wait until all buckets are free to be added, then add them all.
- while (true) {
- bool allOk = true;
- key_type waitingFor(0);
-
- for (uint32_t i=0; i<keys.size(); i++) {
- if (_lockedKeys.exist(LockId(keys[i], clientId))) {
- waitingFor = keys[i];
- allOk = false;
- break;
- }
- }
-
- if (!allOk) {
- typename LockWaiters::Key waitId(_lockWaiters.insert(LockId(waitingFor, clientId)));
- _cond.wait(guard);
- _lockWaiters.erase(waitId);
- } else {
- for (uint32_t i=0; i<keys.size(); i++) {
- typename Map::iterator it = _map.find(keys[i]);
- if (it != _map.end()) {
- _lockedKeys.insert(LockId(keys[i], clientId));
- results[BucketId(BucketId::keyToBucketId(keys[i]))]
- = WrappedEntry(*this, keys[i], it->second,
- clientId, true);
- }
- }
- break;
- }
- }
-}
-
-uint8_t getMinDiffBits(uint16_t minBits, const document::BucketId& a, const document::BucketId& b);
-
-template<typename Map>
-typename LockableMap<Map>::EntryMap
-LockableMap<Map>::getContained(const BucketId& bucket,
- const char* clientId)
-{
- std::unique_lock<std::mutex> guard(_lock);
- std::map<BucketId, WrappedEntry> results;
-
- BucketId result;
- BucketId::Type keyResult;
- BucketId::Type nextKey;
-
- std::vector<BucketId::Type> keys;
-
- if (getMostSpecificMatch(bucket, result, keyResult, nextKey)) {
- keys.push_back(keyResult);
-
- // Find the super buckets for the most specific match
- getAllContaining(result, keys);
- } else {
- // Find the super buckets for the input bucket
- // because getMostSpecificMatch() might not find the most specific
- // match in all cases of inconsistently split buckets
- getAllContaining(bucket, keys);
- }
-
- if (!keys.empty()) {
- addAndLockResults(keys, clientId, results, guard);
- }
-
- return results;
-}
-
-template<typename Map>
-void
-LockableMap<Map>::getAllWithoutLocking(const BucketId& bucket,
- std::vector<BucketId::Type>& keys)
-{
- BucketId result;
- BucketId::Type keyResult;
- BucketId::Type nextKey;
-
- typename Map::iterator it = _map.end();
-
- if (getMostSpecificMatch(bucket, result, keyResult, nextKey)) {
- keys.push_back(keyResult);
-
- // Find the super buckets for the most specific match
- getAllContaining(result, keys);
-
- it = _map.find(keyResult);
- if (it != _map.end()) {
- // Skipping nextKey, since it was equal to keyResult
- ++it;
- }
- } else {
- // Find the super buckets for the input bucket
- // because getMostSpecificMatch() might not find the most specific
- // match in all cases of inconsistently split buckets
- getAllContaining(bucket, keys);
-
- it = _map.find(nextKey);
- if (it != _map.end()) {
- // Nextkey might be contained in the imput bucket,
- // e.g. if it is the first bucket in bucketdb
- BucketId id = BucketId(BucketId::keyToBucketId(it->first));
- if (!bucket.contains(id)) {
- ++it;
- }
- }
- }
-
- // Buckets contained in the found bucket will come immediately after it.
- // Traverse the map to find them.
- for (; it != _map.end(); ++it) {
- BucketId id(BucketId(BucketId::keyToBucketId(it->first)));
-
- if (bucket.contains(id)) {
- keys.push_back(it->first);
- } else {
- break;
- }
- }
-}
-
-/**
- * Returns the given bucket, its super buckets and its sub buckets.
- */
-template<typename Map>
-typename LockableMap<Map>::EntryMap
-LockableMap<Map>::getAll(const BucketId& bucket, const char* clientId)
-{
- std::unique_lock<std::mutex> guard(_lock);
-
- std::map<BucketId, WrappedEntry> results;
- std::vector<BucketId::Type> keys;
-
- getAllWithoutLocking(bucket, keys);
-
- addAndLockResults(keys, clientId, results, guard);
-
- return results;
-}
-
-template<typename Map>
-bool
-LockableMap<Map>::isConsistent(const typename LockableMap<Map>::WrappedEntry& entry)
-{
- std::lock_guard<std::mutex> guard(_lock);
-
- std::vector<BucketId::Type> keys;
-
- getAllWithoutLocking(entry.getBucketId(), keys);
- assert(keys.size() >= 1);
- assert(keys.size() != 1 || keys[0] == entry.getKey());
-
- return keys.size() == 1;
-}
-
-template<typename Map>
-void
-LockableMap<Map>::showLockClients(vespalib::asciistream & out) const
-{
- std::lock_guard<std::mutex> guard(_lock);
- out << "Currently grabbed locks:";
- for (typename LockIdSet::const_iterator it = _lockedKeys.begin();
- it != _lockedKeys.end(); ++it)
- {
- out << "\n "
- << BucketId(BucketId::keyToBucketId(it->_key))
- << " - " << it->_owner;
- }
- out << "\nClients waiting for keys:";
- for (typename LockWaiters::const_iterator it = _lockWaiters.begin();
- it != _lockWaiters.end(); ++it)
- {
- out << "\n "
- << BucketId(BucketId::keyToBucketId(it->second._key))
- << " - " << it->second._owner;
- }
-}
-
-}
diff --git a/storage/src/vespa/storage/bucketdb/storbucketdb.cpp b/storage/src/vespa/storage/bucketdb/storbucketdb.cpp
index 9e692d7d602..b6a5a2320b1 100644
--- a/storage/src/vespa/storage/bucketdb/storbucketdb.cpp
+++ b/storage/src/vespa/storage/bucketdb/storbucketdb.cpp
@@ -2,8 +2,6 @@
#include "storbucketdb.h"
#include "btree_lockable_map.h"
-#include "judymultimap.hpp"
-#include "lockablemap.h"
#include <vespa/log/log.h>
LOG_SETUP(".storage.bucketdb.stor_bucket_db");
@@ -40,10 +38,6 @@ operator<<(std::ostream& out, const StorageBucketInfo& info) {
namespace {
-std::unique_ptr<AbstractBucketMap<StorageBucketInfo>> make_legacy_db_impl() {
- return std::make_unique<LockableMap<JudyMultiMap<StorageBucketInfo>>>();
-}
-
std::unique_ptr<AbstractBucketMap<StorageBucketInfo>> make_btree_db_impl() {
return std::make_unique<BTreeLockableMap<StorageBucketInfo>>();
}
@@ -52,9 +46,8 @@ std::unique_ptr<AbstractBucketMap<StorageBucketInfo>> make_btree_db_impl() {
} // bucketdb
-StorBucketDatabase::StorBucketDatabase(bool use_btree_db)
- : _impl(use_btree_db ? bucketdb::make_btree_db_impl()
- : bucketdb::make_legacy_db_impl())
+StorBucketDatabase::StorBucketDatabase([[maybe_unused]] bool use_btree_db)
+ : _impl(bucketdb::make_btree_db_impl())
{}
void
@@ -142,6 +135,4 @@ StorBucketDatabase::acquire_read_guard() const {
return _impl->acquire_read_guard();
}
-template class JudyMultiMap<bucketdb::StorageBucketInfo>;
-
} // storage
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index 2653391ecfa..8e1ee6cbb3a 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -2,7 +2,6 @@
#include "filestorhandlerimpl.h"
#include "filestormanager.h"
-#include <vespa/storage/bucketdb/lockablemap.hpp>
#include <vespa/storage/bucketdb/minimumusedbitstracker.h>
#include <vespa/storage/common/bucketmessages.h>
#include <vespa/storage/common/content_bucket_space_repo.h>
@@ -18,8 +17,10 @@
#include <vespa/storageapi/message/persistence.h>
#include <vespa/storageapi/message/removelocation.h>
#include <vespa/storageapi/message/stat.h>
+#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <thread>
#include <vespa/log/bufferedlogger.h>
LOG_SETUP(".persistence.filestor.manager");
diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp
index 0885483f674..21cf8a064e1 100644
--- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp
+++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp
@@ -4,7 +4,6 @@
#include <vespa/vespalib/util/programoptions.h>
#include <vespa/vdslib/distribution/distribution.h>
#include <vespa/vdslib/state/clusterstate.h>
-#include <vespa/storage/bucketdb/judyarray.h>
#include <vespa/vespalib/util/stringfmt.h>
#include <iomanip>
#include <iostream>