summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-12 11:58:40 +0000
committerTor Brede Vekterli <vekterli@yahoo-inc.com>2017-12-15 15:20:07 +0000
commita248ff694ca95218c4127f9156735ceb3fbba4d7 (patch)
tree2310e595b1e280c30b33a17d77dd13b7396a48cf
parent77242dcd5594b4b481403491c47979cc866a569c (diff)
Add configurable bucket resolver and fixed space repo
Make default (aka. placeholder) bucket space index 1, not 0. Bucket space index 0 is now considered an invalid space.
-rw-r--r--document/src/vespa/document/bucket/bucketspace.h18
-rw-r--r--document/src/vespa/document/test/make_bucket_space.cpp6
-rw-r--r--persistence/src/tests/spi/CMakeLists.txt1
-rw-r--r--persistence/src/tests/spi/fixed_bucket_spaces_test.cpp64
-rw-r--r--persistence/src/vespa/persistence/spi/CMakeLists.txt13
-rw-r--r--persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp33
-rw-r--r--persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h30
-rw-r--r--searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp2
-rw-r--r--storage/src/tests/distributor/blockingoperationstartertest.cpp2
-rw-r--r--storage/src/tests/distributor/idealstatemanagertest.cpp12
-rw-r--r--storage/src/tests/distributor/maintenanceschedulertest.cpp8
-rw-r--r--storage/src/tests/distributor/pendingmessagetrackertest.cpp6
-rw-r--r--storage/src/tests/distributor/putoperationtest.cpp2
-rw-r--r--storage/src/tests/distributor/simplemaintenancescannertest.cpp18
-rw-r--r--storage/src/tests/distributor/throttlingoperationstartertest.cpp2
-rw-r--r--storage/src/tests/storageserver/CMakeLists.txt1
-rw-r--r--storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp137
-rw-r--r--storage/src/vespa/storage/storageserver/CMakeLists.txt1
-rw-r--r--storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp36
-rw-r--r--storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h36
20 files changed, 385 insertions, 43 deletions
diff --git a/document/src/vespa/document/bucket/bucketspace.h b/document/src/vespa/document/bucket/bucketspace.h
index 1198b173a4b..99b510f7aff 100644
--- a/document/src/vespa/document/bucket/bucketspace.h
+++ b/document/src/vespa/document/bucket/bucketspace.h
@@ -16,15 +16,16 @@ class BucketSpace {
public:
using Type = uint64_t;
- BucketSpace(const BucketSpace&) noexcept = default;
- BucketSpace& operator=(const BucketSpace&) noexcept = default;
- explicit BucketSpace(Type id) noexcept : _id(id) {}
+ constexpr BucketSpace(const BucketSpace&) noexcept = default;
+ constexpr BucketSpace& operator=(const BucketSpace&) noexcept = default;
+ constexpr explicit BucketSpace(Type id) noexcept : _id(id) {}
- bool operator <(const BucketSpace& bucket) const noexcept { return _id < bucket._id; }
- bool operator==(const BucketSpace& bucket) const noexcept { return _id == bucket._id; }
- bool operator!=(const BucketSpace& bucket) const noexcept { return _id != bucket._id; }
+ constexpr bool operator <(const BucketSpace& bucket) const noexcept { return _id < bucket._id; }
+ constexpr bool operator==(const BucketSpace& bucket) const noexcept { return _id == bucket._id; }
+ constexpr bool operator!=(const BucketSpace& bucket) const noexcept { return _id != bucket._id; }
- Type getId() const noexcept { return _id; }
+ constexpr Type getId() const noexcept { return _id; }
+ constexpr bool valid() const noexcept { return (_id != 0); }
vespalib::string toString() const;
struct hash {
@@ -36,7 +37,8 @@ public:
/*
* Temporary placeholder value while wiring in use of BucketSpace in APIs.
*/
- static BucketSpace placeHolder() { return BucketSpace(0); }
+ static constexpr BucketSpace placeHolder() noexcept { return BucketSpace(1); }
+ static constexpr BucketSpace invalid() noexcept { return BucketSpace(0); }
private:
Type _id;
};
diff --git a/document/src/vespa/document/test/make_bucket_space.cpp b/document/src/vespa/document/test/make_bucket_space.cpp
index be8292fcf71..dae0399e75d 100644
--- a/document/src/vespa/document/test/make_bucket_space.cpp
+++ b/document/src/vespa/document/test/make_bucket_space.cpp
@@ -11,12 +11,12 @@ BucketSpace makeBucketSpace()
BucketSpace makeBucketSpace(const vespalib::string &docTypeName)
{
- // Used by persistence conformance test to map fron document type name
+ // Used by persistence conformance test to map from document type name
// to bucket space. See document::TestDocRepo for known document types.
if (docTypeName == "no") {
- return BucketSpace(2);
+ return BucketSpace(3);
} else if (docTypeName == "testdoctype2") {
- return BucketSpace(1);
+ return BucketSpace(2);
} else {
return makeBucketSpace();
}
diff --git a/persistence/src/tests/spi/CMakeLists.txt b/persistence/src/tests/spi/CMakeLists.txt
index a130573e028..c51270a420c 100644
--- a/persistence/src/tests/spi/CMakeLists.txt
+++ b/persistence/src/tests/spi/CMakeLists.txt
@@ -2,6 +2,7 @@
vespa_add_library(persistence_testspi
SOURCES
clusterstatetest.cpp
+ fixed_bucket_spaces_test.cpp
DEPENDS
persistence_persistence_conformancetest
persistence
diff --git a/persistence/src/tests/spi/fixed_bucket_spaces_test.cpp b/persistence/src/tests/spi/fixed_bucket_spaces_test.cpp
new file mode 100644
index 00000000000..7e36d80248a
--- /dev/null
+++ b/persistence/src/tests/spi/fixed_bucket_spaces_test.cpp
@@ -0,0 +1,64 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/persistence/spi/fixed_bucket_spaces.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace storage::spi {
+
+struct FixedBucketSpacesTest : CppUnit::TestFixture {
+ CPPUNIT_TEST_SUITE(FixedBucketSpacesTest);
+ CPPUNIT_TEST(bucket_space_from_name_is_defined_for_default_space);
+ CPPUNIT_TEST(bucket_space_from_name_is_defined_for_global_space);
+ CPPUNIT_TEST(bucket_space_from_name_throws_exception_for_unknown_space);
+ CPPUNIT_TEST(name_from_bucket_space_is_defined_for_default_space);
+ CPPUNIT_TEST(name_from_bucket_space_is_defined_for_global_space);
+ CPPUNIT_TEST(name_from_bucket_space_throws_exception_for_unknown_space);
+ CPPUNIT_TEST_SUITE_END();
+
+ void bucket_space_from_name_is_defined_for_default_space();
+ void bucket_space_from_name_is_defined_for_global_space();
+ void bucket_space_from_name_throws_exception_for_unknown_space();
+ void name_from_bucket_space_is_defined_for_default_space();
+ void name_from_bucket_space_is_defined_for_global_space();
+ void name_from_bucket_space_throws_exception_for_unknown_space();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(FixedBucketSpacesTest);
+
+using document::BucketSpace;
+
+void FixedBucketSpacesTest::bucket_space_from_name_is_defined_for_default_space() {
+ CPPUNIT_ASSERT_EQUAL(FixedBucketSpaces::default_space(), FixedBucketSpaces::from_string("default"));
+}
+
+void FixedBucketSpacesTest::bucket_space_from_name_is_defined_for_global_space() {
+ CPPUNIT_ASSERT_EQUAL(FixedBucketSpaces::global_space(), FixedBucketSpaces::from_string("global"));
+}
+
+void FixedBucketSpacesTest::bucket_space_from_name_throws_exception_for_unknown_space() {
+ try {
+ FixedBucketSpaces::from_string("banana");
+ CPPUNIT_FAIL("Expected exception on unknown bucket space name");
+ } catch (spi::UnknownBucketSpaceException& e) {
+ }
+}
+
+void FixedBucketSpacesTest::name_from_bucket_space_is_defined_for_default_space() {
+ CPPUNIT_ASSERT_EQUAL(vespalib::stringref("default"),
+ FixedBucketSpaces::to_string(FixedBucketSpaces::default_space()));
+}
+
+void FixedBucketSpacesTest::name_from_bucket_space_is_defined_for_global_space() {
+ CPPUNIT_ASSERT_EQUAL(vespalib::stringref("global"),
+ FixedBucketSpaces::to_string(FixedBucketSpaces::global_space()));
+}
+
+void FixedBucketSpacesTest::name_from_bucket_space_throws_exception_for_unknown_space() {
+ try {
+ FixedBucketSpaces::to_string(BucketSpace(4567));
+ CPPUNIT_FAIL("Expected exception on unknown bucket space value");
+ } catch (spi::UnknownBucketSpaceException& e) {
+ }
+}
+
+} \ No newline at end of file
diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt
index a8b1faadcd3..a2b8fa7a79c 100644
--- a/persistence/src/vespa/persistence/spi/CMakeLists.txt
+++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt
@@ -1,19 +1,20 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(persistence_spi OBJECT
SOURCES
+ abstractpersistenceprovider.cpp
bucket.cpp
bucketinfo.cpp
- exceptions.cpp
- persistenceprovider.cpp
- partitionstate.cpp
- abstractpersistenceprovider.cpp
clusterstate.cpp
context.cpp
+ docentry.cpp
+ exceptions.cpp
+ fixed_bucket_spaces.cpp
metricpersistenceprovider.cpp
+ partitionstate.cpp
+ persistenceprovider.cpp
read_consistency.cpp
- result
+ result.cpp
selection.cpp
test.cpp
- docentry
DEPENDS
)
diff --git a/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp
new file mode 100644
index 00000000000..6a8ec0f18f7
--- /dev/null
+++ b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "fixed_bucket_spaces.h"
+
+namespace storage::spi {
+
+VESPA_IMPLEMENT_EXCEPTION(UnknownBucketSpaceException, vespalib::IllegalArgumentException)
+
+// Some sanity checks to ensure we don't mess up any legacy mappings.
+static_assert(document::BucketSpace::placeHolder() != document::BucketSpace::invalid());
+static_assert(FixedBucketSpaces::default_space() == document::BucketSpace::placeHolder());
+static_assert(FixedBucketSpaces::global_space() != FixedBucketSpaces::default_space());
+
+document::BucketSpace FixedBucketSpaces::from_string(vespalib::stringref name) {
+ if (name == "default") {
+ return default_space();
+ } else if (name == "global") {
+ return global_space();
+ } else {
+ throw UnknownBucketSpaceException("Unknown bucket space name: " + vespalib::string(name), VESPA_STRLOC);
+ }
+}
+
+vespalib::stringref FixedBucketSpaces::to_string(document::BucketSpace space) {
+ if (space == default_space()) {
+ return "default";
+ } else if (space == global_space()) {
+ return "global";
+ } else {
+ throw UnknownBucketSpaceException("Unknown bucket space: " + space.toString(), VESPA_STRLOC);
+ }
+}
+
+}
diff --git a/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h
new file mode 100644
index 00000000000..c2e97407797
--- /dev/null
+++ b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h
@@ -0,0 +1,30 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/bucket/bucketspace.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/stllike/string.h>
+
+namespace storage::spi {
+
+VESPA_DEFINE_EXCEPTION(UnknownBucketSpaceException, vespalib::IllegalArgumentException);
+
+/**
+ * Minimal repository/factory of bucket spaces hard coded for default and global
+ * distributions.
+ */
+struct FixedBucketSpaces {
+ static constexpr document::BucketSpace default_space() { return document::BucketSpace(1); };
+ static constexpr document::BucketSpace global_space() { return document::BucketSpace(2); }
+
+ // Post-condition: returned space has valid() == true iff name
+ // is either "default" or "global".
+ // Throws UnknownBucketSpaceException if name does not map to a known bucket space.
+ static document::BucketSpace from_string(vespalib::stringref name);
+ // Post-condition: returned string can be losslessly passed to from_string()
+ // iff space is equal to default_space() or global_space().
+ // Throws UnknownBucketSpaceException if space does not map to a known name.
+ static vespalib::stringref to_string(document::BucketSpace space);
+};
+
+}
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
index 4a195514db1..bd274093f87 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -369,7 +369,7 @@ Timestamp tstamp2(2);
Timestamp tstamp3(3);
DocumentSelection doc_sel("");
Selection selection(doc_sel);
-BucketSpace altBucketSpace(1);
+BucketSpace altBucketSpace(2);
void
diff --git a/storage/src/tests/distributor/blockingoperationstartertest.cpp b/storage/src/tests/distributor/blockingoperationstartertest.cpp
index c2fdc25cebf..0160f5c9e51 100644
--- a/storage/src/tests/distributor/blockingoperationstartertest.cpp
+++ b/storage/src/tests/distributor/blockingoperationstartertest.cpp
@@ -60,7 +60,7 @@ BlockingOperationStarterTest::testOperationNotBlockedWhenNoMessagesPending()
{
CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(),
OperationStarter::Priority(0)));
- CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 0\n"),
+ CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 0\n"),
_starterImpl->toString());
}
diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp
index 0c695f9a3d4..bca15d702f5 100644
--- a/storage/src/tests/distributor/idealstatemanagertest.cpp
+++ b/storage/src/tests/distributor/idealstatemanagertest.cpp
@@ -143,9 +143,9 @@ IdealStateManagerTest::testClearActiveOnNodeDown()
}
CPPUNIT_ASSERT_EQUAL(
- std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"
- "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)) (pri 100)\n"
- "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)) (pri 100)\n"),
+ std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"
+ "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)) (pri 100)\n"
+ "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)) (pri 100)\n"),
_distributor->getActiveIdealStateOperations());
setSystemState(lib::ClusterState("distributor:1 storage:3 .0.s:d"));
@@ -169,19 +169,19 @@ IdealStateManagerTest::testRecheckWhenActive()
tick();
CPPUNIT_ASSERT_EQUAL(
- std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"),
+ std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"),
_distributor->getActiveIdealStateOperations());
tick();
CPPUNIT_ASSERT_EQUAL(
- std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"),
+ std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"),
_distributor->getActiveIdealStateOperations());
tick();
CPPUNIT_ASSERT_EQUAL(
- std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"),
+ std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"),
_distributor->getActiveIdealStateOperations());
}
diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp
index 7e3d92053f8..db0347617f0 100644
--- a/storage/src/tests/distributor/maintenanceschedulertest.cpp
+++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp
@@ -70,7 +70,7 @@ MaintenanceSchedulerTest::testOperationIsScheduled()
{
_priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::MEDIUM));
_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE);
- CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 100\n"),
+ CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 100\n"),
_operationStarter->toString());
}
@@ -89,9 +89,9 @@ MaintenanceSchedulerTest::testSuppressLowPrioritiesInEmergencyMode()
_priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 2)), Priority::VERY_HIGH));
CPPUNIT_ASSERT_EQUAL(WaitTimeMs(0), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE));
CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE));
- CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri 0\n"),
+ CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri 0\n"),
_operationStarter->toString());
- CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri HIGH)\n"),
+ CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGH)\n"),
_priorityDb->toString());
}
@@ -102,7 +102,7 @@ MaintenanceSchedulerTest::testPriorityNotClearedIfOperationNotStarted()
_operationStarter->setShouldStartOperations(false);
WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE));
CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), waitMs);
- CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri HIGH)\n"),
+ CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGH)\n"),
_priorityDb->toString());
}
diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp
index cca55a11b38..7adadd226d7 100644
--- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp
+++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp
@@ -254,7 +254,7 @@ PendingMessageTrackerTest::testSimple()
CPPUNIT_ASSERT_CONTAIN(
std::string(
- "<b>Bucket(BucketSpace(0x0000000000000000), BucketId(0x40000000000004d2))</b>\n"
+ "<b>Bucket(BucketSpace(0x0000000000000001), BucketId(0x40000000000004d2))</b>\n"
"<ul>\n"
"<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> "
"Remove(BucketId(0x40000000000004d2), "
@@ -341,14 +341,14 @@ PendingMessageTrackerTest::testMultipleMessages()
CPPUNIT_ASSERT_CONTAIN(
std::string(
- "<b>Bucket(BucketSpace(0x0000000000000000), BucketId(0x40000000000004d2))</b>\n"
+ "<b>Bucket(BucketSpace(0x0000000000000001), BucketId(0x40000000000004d2))</b>\n"
"<ul>\n"
"<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:0, timestamp 1000)</li>\n"
"<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:2, timestamp 1002)</li>\n"
"<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:1, timestamp 1001)</li>\n"
"<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:3, timestamp 1003)</li>\n"
"</ul>\n"
- "<b>Bucket(BucketSpace(0x0000000000000000), BucketId(0x40000000000011d7))</b>\n"
+ "<b>Bucket(BucketSpace(0x0000000000000001), BucketId(0x40000000000011d7))</b>\n"
"<ul>\n"
"<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:0, timestamp 2000)</li>\n"
"<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:2, timestamp 2002)</li>\n"
diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp
index 7f54e163006..e621ef8645c 100644
--- a/storage/src/tests/distributor/putoperationtest.cpp
+++ b/storage/src/tests/distributor/putoperationtest.cpp
@@ -282,7 +282,7 @@ PutOperationTest::testNodeRemovedOnReply()
CPPUNIT_ASSERT_EQUAL(std::string(
"PutReply(doc:test:test, BucketId(0x0000000000000000), "
"timestamp 100) ReturnCode(BUCKET_DELETED, "
- "Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000008b13)) was deleted from nodes [0] "
+ "Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000008b13)) was deleted from nodes [0] "
"after message was sent but before it was done. "
"Sent to [1,0])"),
_sender.getLastReply());
diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp
index 66a2d3efa6c..394df6024fd 100644
--- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp
+++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp
@@ -92,7 +92,7 @@ void
SimpleMaintenanceScannerTest::testPrioritizeSingleBucket()
{
addBucketToDb(1);
- std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n");
+ std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n");
auto scanResult = _scanner->scanNext();
CPPUNIT_ASSERT(!scanResult.isDone());
@@ -141,9 +141,9 @@ SimpleMaintenanceScannerTest::testPrioritizeMultipleBuckets()
addBucketToDb(1);
addBucketToDb(2);
addBucketToDb(3);
- std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"
- "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri VERY_HIGH)\n"
- "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n");
+ std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"
+ "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri VERY_HIGH)\n"
+ "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)), pri VERY_HIGH)\n");
CPPUNIT_ASSERT(scanEntireDatabase(3));
CPPUNIT_ASSERT_EQUAL(sortLines(expected),
@@ -168,8 +168,8 @@ SimpleMaintenanceScannerTest::testReset()
addBucketToDb(3);
CPPUNIT_ASSERT(scanEntireDatabase(2));
- std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"
- "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n");
+ std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"
+ "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)), pri VERY_HIGH)\n");
CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString());
addBucketToDb(2);
@@ -179,9 +179,9 @@ SimpleMaintenanceScannerTest::testReset()
_scanner->reset();
CPPUNIT_ASSERT(scanEntireDatabase(3));
- expected = "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"
- "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri VERY_HIGH)\n"
- "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n";
+ expected = "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"
+ "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri VERY_HIGH)\n"
+ "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)), pri VERY_HIGH)\n";
CPPUNIT_ASSERT_EQUAL(sortLines(expected), sortLines(_priorityDb->toString()));
}
diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp
index c3aebcafe06..c3290a8c0f6 100644
--- a/storage/src/tests/distributor/throttlingoperationstartertest.cpp
+++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp
@@ -70,7 +70,7 @@ ThrottlingOperationStarterTest::testOperationStartingIsForwardedToImplementation
{
CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(),
OperationStarter::Priority(0)));
- CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 0\n"),
+ CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 0\n"),
_starterImpl->toString());
}
diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt
index 38fb0f6235a..95faf7e433e 100644
--- a/storage/src/tests/storageserver/CMakeLists.txt
+++ b/storage/src/tests/storageserver/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_library(storage_teststorageserver TEST
bucketintegritycheckertest.cpp
changedbucketownershiphandlertest.cpp
communicationmanagertest.cpp
+ configurable_bucket_resolver_test.cpp
documentapiconvertertest.cpp
mergethrottlertest.cpp
priorityconvertertest.cpp
diff --git a/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp
new file mode 100644
index 00000000000..3f121240065
--- /dev/null
+++ b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp
@@ -0,0 +1,137 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/storage/storageserver/configurable_bucket_resolver.h>
+#include <vespa/document/base/documentid.h>
+#include <vespa/persistence/spi/fixed_bucket_spaces.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+namespace storage {
+
+using document::DocumentId;
+
+struct ConfigurableBucketResolverTest : CppUnit::TestFixture {
+ CPPUNIT_TEST_SUITE(ConfigurableBucketResolverTest);
+ CPPUNIT_TEST(bucket_space_from_name_is_defined_for_default_space);
+ CPPUNIT_TEST(bucket_space_from_name_is_defined_for_global_space);
+ CPPUNIT_TEST(bucket_space_from_name_throws_exception_for_unknown_space);
+ CPPUNIT_TEST(name_from_bucket_space_is_defined_for_default_space);
+ CPPUNIT_TEST(name_from_bucket_space_is_defined_for_global_space);
+ CPPUNIT_TEST(name_from_bucket_space_throws_exception_for_unknown_space);
+ CPPUNIT_TEST(known_bucket_space_is_resolved_from_document_id);
+ CPPUNIT_TEST(unknown_bucket_space_in_id_throws_exception);
+ CPPUNIT_TEST(can_create_resolver_from_bucket_space_config);
+ CPPUNIT_TEST_SUITE_END();
+
+ using BucketSpaceMapping = ConfigurableBucketResolver::BucketSpaceMapping;
+
+ BucketSpaceMapping create_simple_mapping() {
+ return {{"foo", spi::FixedBucketSpaces::default_space()},
+ {"bar", spi::FixedBucketSpaces::default_space()},
+ {"baz", spi::FixedBucketSpaces::global_space()}};
+ }
+
+ ConfigurableBucketResolver create_empty_resolver() {
+ return ConfigurableBucketResolver({});
+ }
+
+ ConfigurableBucketResolver create_simple_resolver() {
+ return ConfigurableBucketResolver(create_simple_mapping());
+ }
+
+ void bucket_space_from_name_is_defined_for_default_space();
+ void bucket_space_from_name_is_defined_for_global_space();
+ void bucket_space_from_name_throws_exception_for_unknown_space();
+ void name_from_bucket_space_is_defined_for_default_space();
+ void name_from_bucket_space_is_defined_for_global_space();
+ void name_from_bucket_space_throws_exception_for_unknown_space();
+ void known_bucket_space_is_resolved_from_document_id();
+ void unknown_bucket_space_in_id_throws_exception();
+ void can_create_resolver_from_bucket_space_config();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ConfigurableBucketResolverTest);
+
+// TODO reduce overlap with FixedBucketSpacesTest
+void ConfigurableBucketResolverTest::bucket_space_from_name_is_defined_for_default_space() {
+ auto space = create_empty_resolver().bucketSpaceFromName("default");
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(), space);
+}
+
+void ConfigurableBucketResolverTest::bucket_space_from_name_is_defined_for_global_space() {
+ auto space = create_empty_resolver().bucketSpaceFromName("global");
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(), space);
+}
+
+void ConfigurableBucketResolverTest::bucket_space_from_name_throws_exception_for_unknown_space() {
+ try {
+ create_empty_resolver().bucketSpaceFromName("bjarne");
+ CPPUNIT_FAIL("Expected exception on unknown bucket space name");
+ } catch (spi::UnknownBucketSpaceException& e) {
+ }
+}
+
+void ConfigurableBucketResolverTest::name_from_bucket_space_is_defined_for_default_space() {
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("default"),
+ create_empty_resolver().nameFromBucketSpace(spi::FixedBucketSpaces::default_space()));
+}
+
+void ConfigurableBucketResolverTest::name_from_bucket_space_is_defined_for_global_space() {
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("global"),
+ create_empty_resolver().nameFromBucketSpace(spi::FixedBucketSpaces::global_space()));
+}
+
+void ConfigurableBucketResolverTest::name_from_bucket_space_throws_exception_for_unknown_space() {
+ try {
+ create_empty_resolver().nameFromBucketSpace(document::BucketSpace(1234));
+ CPPUNIT_FAIL("Expected exception on unknown bucket space value");
+ } catch (spi::UnknownBucketSpaceException& e) {
+ }
+}
+
+void ConfigurableBucketResolverTest::known_bucket_space_is_resolved_from_document_id() {
+ auto resolver = create_simple_resolver();
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(),
+ resolver.bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace());
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(),
+ resolver.bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace());
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(),
+ resolver.bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace());
+}
+
+void ConfigurableBucketResolverTest::unknown_bucket_space_in_id_throws_exception() {
+ try {
+ create_simple_resolver().bucketFromId(DocumentId("id::bjarne::xyz"));
+ CPPUNIT_FAIL("Expected exception on unknown document type -> bucket space mapping");
+ } catch (spi::UnknownBucketSpaceException& e) {
+ }
+}
+
+using BucketSpacesConfigBuilder = vespa::config::content::core::BucketspacesConfigBuilder;
+
+namespace {
+
+BucketSpacesConfigBuilder::Documenttype make_doc_type(vespalib::stringref name, vespalib::stringref space) {
+ BucketSpacesConfigBuilder::Documenttype doc_type;
+ doc_type.name = name;
+ doc_type.bucketspace = space;
+ return doc_type;
+}
+
+}
+
+void ConfigurableBucketResolverTest::can_create_resolver_from_bucket_space_config() {
+ BucketSpacesConfigBuilder builder;
+ builder.documenttype.emplace_back(make_doc_type("foo", "default"));
+ builder.documenttype.emplace_back(make_doc_type("bar", "global"));
+ builder.documenttype.emplace_back(make_doc_type("baz", "global"));
+ auto resolver = ConfigurableBucketResolver::from_config(builder);
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(),
+ resolver->bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace());
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(),
+ resolver->bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace());
+ CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(),
+ resolver->bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace());
+}
+
+}
+
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index c0238922a91..4fb3a5a0b99 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -6,6 +6,7 @@ vespa_add_library(storage_storageserver
changedbucketownershiphandler.cpp
communicationmanager.cpp
communicationmanagermetrics.cpp
+ configurable_bucket_resolver.cpp
distributornode.cpp
distributornodecontext.cpp
documentapiconverter.cpp
diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp
new file mode 100644
index 00000000000..86c802a65cf
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/document/base/documentid.h>
+#include <vespa/persistence/spi/fixed_bucket_spaces.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include "configurable_bucket_resolver.h"
+
+namespace storage {
+
+document::Bucket ConfigurableBucketResolver::bucketFromId(const document::DocumentId& id) const {
+ auto iter = _type_to_space.find(id.getDocType());
+ if (iter != _type_to_space.end()) {
+ return document::Bucket(iter->second, document::BucketId(0));
+ }
+ throw spi::UnknownBucketSpaceException("Unknown bucket space mapping for document type '"
+ + id.getDocType() + "' in id: " + id.toString(), VESPA_STRLOC);
+}
+
+document::BucketSpace ConfigurableBucketResolver::bucketSpaceFromName(const vespalib::string& name) const {
+ return spi::FixedBucketSpaces::from_string(name);
+}
+
+vespalib::string ConfigurableBucketResolver::nameFromBucketSpace(const document::BucketSpace& space) const {
+ return spi::FixedBucketSpaces::to_string(space);
+}
+
+std::shared_ptr<ConfigurableBucketResolver> ConfigurableBucketResolver::from_config(
+ const vespa::config::content::core::BucketspacesConfig& config) {
+ ConfigurableBucketResolver::BucketSpaceMapping type_to_space;
+ for (auto& mapping : config.documenttype) {
+ type_to_space.emplace(mapping.name, spi::FixedBucketSpaces::from_string(mapping.bucketspace));
+ }
+ return std::make_shared<ConfigurableBucketResolver>(std::move(type_to_space));
+}
+
+}
diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h
new file mode 100644
index 00000000000..acebd9777fb
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/storage/config/config-bucketspaces.h>
+#include <vespa/storage/common/bucket_resolver.h>
+#include <vespa/vespalib/stllike/hash_fun.h>
+#include <memory>
+#include <unordered_map>
+
+namespace storage {
+
+/**
+ * Immutable implementation of BucketResolver which maintains an explicit
+ * mapping from document type to bucket space.
+ *
+ * If an unknown document type or bucket space is given as an argument,
+ * an spi::UnknownBucketSpaceException is thrown.
+ */
+class ConfigurableBucketResolver : public BucketResolver {
+public:
+ using BucketSpaceMapping = std::unordered_map<vespalib::string, document::BucketSpace, vespalib::hash<vespalib::string>>;
+ const BucketSpaceMapping _type_to_space;
+public:
+ explicit ConfigurableBucketResolver(BucketSpaceMapping type_to_space)
+ : _type_to_space(std::move(type_to_space))
+ {}
+
+ document::Bucket bucketFromId(const document::DocumentId&) const override;
+ document::BucketSpace bucketSpaceFromName(const vespalib::string& name) const override;
+ vespalib::string nameFromBucketSpace(const document::BucketSpace& space) const override;
+
+ static std::shared_ptr<ConfigurableBucketResolver> from_config(
+ const vespa::config::content::core::BucketspacesConfig& config);
+};
+
+} \ No newline at end of file