diff options
author | Tor Brede Vekterli <vekterli@yahoo-inc.com> | 2017-12-12 11:58:40 +0000 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@yahoo-inc.com> | 2017-12-15 15:20:07 +0000 |
commit | a248ff694ca95218c4127f9156735ceb3fbba4d7 (patch) | |
tree | 2310e595b1e280c30b33a17d77dd13b7396a48cf | |
parent | 77242dcd5594b4b481403491c47979cc866a569c (diff) |
Add configurable bucket resolver and fixed space repo
Make default (aka. placeholder) bucket space index 1, not 0.
Bucket space index 0 is now considered an invalid space.
20 files changed, 385 insertions, 43 deletions
diff --git a/document/src/vespa/document/bucket/bucketspace.h b/document/src/vespa/document/bucket/bucketspace.h index 1198b173a4b..99b510f7aff 100644 --- a/document/src/vespa/document/bucket/bucketspace.h +++ b/document/src/vespa/document/bucket/bucketspace.h @@ -16,15 +16,16 @@ class BucketSpace { public: using Type = uint64_t; - BucketSpace(const BucketSpace&) noexcept = default; - BucketSpace& operator=(const BucketSpace&) noexcept = default; - explicit BucketSpace(Type id) noexcept : _id(id) {} + constexpr BucketSpace(const BucketSpace&) noexcept = default; + constexpr BucketSpace& operator=(const BucketSpace&) noexcept = default; + constexpr explicit BucketSpace(Type id) noexcept : _id(id) {} - bool operator <(const BucketSpace& bucket) const noexcept { return _id < bucket._id; } - bool operator==(const BucketSpace& bucket) const noexcept { return _id == bucket._id; } - bool operator!=(const BucketSpace& bucket) const noexcept { return _id != bucket._id; } + constexpr bool operator <(const BucketSpace& bucket) const noexcept { return _id < bucket._id; } + constexpr bool operator==(const BucketSpace& bucket) const noexcept { return _id == bucket._id; } + constexpr bool operator!=(const BucketSpace& bucket) const noexcept { return _id != bucket._id; } - Type getId() const noexcept { return _id; } + constexpr Type getId() const noexcept { return _id; } + constexpr bool valid() const noexcept { return (_id != 0); } vespalib::string toString() const; struct hash { @@ -36,7 +37,8 @@ public: /* * Temporary placeholder value while wiring in use of BucketSpace in APIs. */ - static BucketSpace placeHolder() { return BucketSpace(0); } + static constexpr BucketSpace placeHolder() noexcept { return BucketSpace(1); } + static constexpr BucketSpace invalid() noexcept { return BucketSpace(0); } private: Type _id; }; diff --git a/document/src/vespa/document/test/make_bucket_space.cpp b/document/src/vespa/document/test/make_bucket_space.cpp index be8292fcf71..dae0399e75d 100644 --- a/document/src/vespa/document/test/make_bucket_space.cpp +++ b/document/src/vespa/document/test/make_bucket_space.cpp @@ -11,12 +11,12 @@ BucketSpace makeBucketSpace() BucketSpace makeBucketSpace(const vespalib::string &docTypeName) { - // Used by persistence conformance test to map fron document type name + // Used by persistence conformance test to map from document type name // to bucket space. See document::TestDocRepo for known document types. if (docTypeName == "no") { - return BucketSpace(2); + return BucketSpace(3); } else if (docTypeName == "testdoctype2") { - return BucketSpace(1); + return BucketSpace(2); } else { return makeBucketSpace(); } diff --git a/persistence/src/tests/spi/CMakeLists.txt b/persistence/src/tests/spi/CMakeLists.txt index a130573e028..c51270a420c 100644 --- a/persistence/src/tests/spi/CMakeLists.txt +++ b/persistence/src/tests/spi/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(persistence_testspi SOURCES clusterstatetest.cpp + fixed_bucket_spaces_test.cpp DEPENDS persistence_persistence_conformancetest persistence diff --git a/persistence/src/tests/spi/fixed_bucket_spaces_test.cpp b/persistence/src/tests/spi/fixed_bucket_spaces_test.cpp new file mode 100644 index 00000000000..7e36d80248a --- /dev/null +++ b/persistence/src/tests/spi/fixed_bucket_spaces_test.cpp @@ -0,0 +1,64 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/persistence/spi/fixed_bucket_spaces.h> +#include <cppunit/extensions/HelperMacros.h> + +namespace storage::spi { + +struct FixedBucketSpacesTest : CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(FixedBucketSpacesTest); + CPPUNIT_TEST(bucket_space_from_name_is_defined_for_default_space); + CPPUNIT_TEST(bucket_space_from_name_is_defined_for_global_space); + CPPUNIT_TEST(bucket_space_from_name_throws_exception_for_unknown_space); + CPPUNIT_TEST(name_from_bucket_space_is_defined_for_default_space); + CPPUNIT_TEST(name_from_bucket_space_is_defined_for_global_space); + CPPUNIT_TEST(name_from_bucket_space_throws_exception_for_unknown_space); + CPPUNIT_TEST_SUITE_END(); + + void bucket_space_from_name_is_defined_for_default_space(); + void bucket_space_from_name_is_defined_for_global_space(); + void bucket_space_from_name_throws_exception_for_unknown_space(); + void name_from_bucket_space_is_defined_for_default_space(); + void name_from_bucket_space_is_defined_for_global_space(); + void name_from_bucket_space_throws_exception_for_unknown_space(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(FixedBucketSpacesTest); + +using document::BucketSpace; + +void FixedBucketSpacesTest::bucket_space_from_name_is_defined_for_default_space() { + CPPUNIT_ASSERT_EQUAL(FixedBucketSpaces::default_space(), FixedBucketSpaces::from_string("default")); +} + +void FixedBucketSpacesTest::bucket_space_from_name_is_defined_for_global_space() { + CPPUNIT_ASSERT_EQUAL(FixedBucketSpaces::global_space(), FixedBucketSpaces::from_string("global")); +} + +void FixedBucketSpacesTest::bucket_space_from_name_throws_exception_for_unknown_space() { + try { + FixedBucketSpaces::from_string("banana"); + CPPUNIT_FAIL("Expected exception on unknown bucket space name"); + } catch (spi::UnknownBucketSpaceException& e) { + } +} + +void FixedBucketSpacesTest::name_from_bucket_space_is_defined_for_default_space() { + CPPUNIT_ASSERT_EQUAL(vespalib::stringref("default"), + FixedBucketSpaces::to_string(FixedBucketSpaces::default_space())); +} + +void FixedBucketSpacesTest::name_from_bucket_space_is_defined_for_global_space() { + CPPUNIT_ASSERT_EQUAL(vespalib::stringref("global"), + FixedBucketSpaces::to_string(FixedBucketSpaces::global_space())); +} + +void FixedBucketSpacesTest::name_from_bucket_space_throws_exception_for_unknown_space() { + try { + FixedBucketSpaces::to_string(BucketSpace(4567)); + CPPUNIT_FAIL("Expected exception on unknown bucket space value"); + } catch (spi::UnknownBucketSpaceException& e) { + } +} + +}
\ No newline at end of file diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt index a8b1faadcd3..a2b8fa7a79c 100644 --- a/persistence/src/vespa/persistence/spi/CMakeLists.txt +++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt @@ -1,19 +1,20 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(persistence_spi OBJECT SOURCES + abstractpersistenceprovider.cpp bucket.cpp bucketinfo.cpp - exceptions.cpp - persistenceprovider.cpp - partitionstate.cpp - abstractpersistenceprovider.cpp clusterstate.cpp context.cpp + docentry.cpp + exceptions.cpp + fixed_bucket_spaces.cpp metricpersistenceprovider.cpp + partitionstate.cpp + persistenceprovider.cpp read_consistency.cpp - result + result.cpp selection.cpp test.cpp - docentry DEPENDS ) diff --git a/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp new file mode 100644 index 00000000000..6a8ec0f18f7 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.cpp @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "fixed_bucket_spaces.h" + +namespace storage::spi { + +VESPA_IMPLEMENT_EXCEPTION(UnknownBucketSpaceException, vespalib::IllegalArgumentException) + +// Some sanity checks to ensure we don't mess up any legacy mappings. +static_assert(document::BucketSpace::placeHolder() != document::BucketSpace::invalid()); +static_assert(FixedBucketSpaces::default_space() == document::BucketSpace::placeHolder()); +static_assert(FixedBucketSpaces::global_space() != FixedBucketSpaces::default_space()); + +document::BucketSpace FixedBucketSpaces::from_string(vespalib::stringref name) { + if (name == "default") { + return default_space(); + } else if (name == "global") { + return global_space(); + } else { + throw UnknownBucketSpaceException("Unknown bucket space name: " + vespalib::string(name), VESPA_STRLOC); + } +} + +vespalib::stringref FixedBucketSpaces::to_string(document::BucketSpace space) { + if (space == default_space()) { + return "default"; + } else if (space == global_space()) { + return "global"; + } else { + throw UnknownBucketSpaceException("Unknown bucket space: " + space.toString(), VESPA_STRLOC); + } +} + +} diff --git a/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h new file mode 100644 index 00000000000..c2e97407797 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/fixed_bucket_spaces.h @@ -0,0 +1,30 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/bucket/bucketspace.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/string.h> + +namespace storage::spi { + +VESPA_DEFINE_EXCEPTION(UnknownBucketSpaceException, vespalib::IllegalArgumentException); + +/** + * Minimal repository/factory of bucket spaces hard coded for default and global + * distributions. + */ +struct FixedBucketSpaces { + static constexpr document::BucketSpace default_space() { return document::BucketSpace(1); }; + static constexpr document::BucketSpace global_space() { return document::BucketSpace(2); } + + // Post-condition: returned space has valid() == true iff name + // is either "default" or "global". + // Throws UnknownBucketSpaceException if name does not map to a known bucket space. + static document::BucketSpace from_string(vespalib::stringref name); + // Post-condition: returned string can be losslessly passed to from_string() + // iff space is equal to default_space() or global_space(). + // Throws UnknownBucketSpaceException if space does not map to a known name. + static vespalib::stringref to_string(document::BucketSpace space); +}; + +} diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 4a195514db1..bd274093f87 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -369,7 +369,7 @@ Timestamp tstamp2(2); Timestamp tstamp3(3); DocumentSelection doc_sel(""); Selection selection(doc_sel); -BucketSpace altBucketSpace(1); +BucketSpace altBucketSpace(2); void diff --git a/storage/src/tests/distributor/blockingoperationstartertest.cpp b/storage/src/tests/distributor/blockingoperationstartertest.cpp index c2fdc25cebf..0160f5c9e51 100644 --- a/storage/src/tests/distributor/blockingoperationstartertest.cpp +++ b/storage/src/tests/distributor/blockingoperationstartertest.cpp @@ -60,7 +60,7 @@ BlockingOperationStarterTest::testOperationNotBlockedWhenNoMessagesPending() { CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); - CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 0\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 0\n"), _starterImpl->toString()); } diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index 0c695f9a3d4..bca15d702f5 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -143,9 +143,9 @@ IdealStateManagerTest::testClearActiveOnNodeDown() } CPPUNIT_ASSERT_EQUAL( - std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n" - "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)) (pri 100)\n" - "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)) (pri 100)\n"), + std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n" + "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)) (pri 100)\n" + "setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)) (pri 100)\n"), _distributor->getActiveIdealStateOperations()); setSystemState(lib::ClusterState("distributor:1 storage:3 .0.s:d")); @@ -169,19 +169,19 @@ IdealStateManagerTest::testRecheckWhenActive() tick(); CPPUNIT_ASSERT_EQUAL( - std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"), + std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"), _distributor->getActiveIdealStateOperations()); tick(); CPPUNIT_ASSERT_EQUAL( - std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"), + std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"), _distributor->getActiveIdealStateOperations()); tick(); CPPUNIT_ASSERT_EQUAL( - std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)) (pri 100)\n"), + std::string("setbucketstate to [0] Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)) (pri 100)\n"), _distributor->getActiveIdealStateOperations()); } diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp index 7e3d92053f8..db0347617f0 100644 --- a/storage/src/tests/distributor/maintenanceschedulertest.cpp +++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp @@ -70,7 +70,7 @@ MaintenanceSchedulerTest::testOperationIsScheduled() { _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::MEDIUM)); _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); - CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 100\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 100\n"), _operationStarter->toString()); } @@ -89,9 +89,9 @@ MaintenanceSchedulerTest::testSuppressLowPrioritiesInEmergencyMode() _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 2)), Priority::VERY_HIGH)); CPPUNIT_ASSERT_EQUAL(WaitTimeMs(0), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); - CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri 0\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri 0\n"), _operationStarter->toString()); - CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri HIGH)\n"), + CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGH)\n"), _priorityDb->toString()); } @@ -102,7 +102,7 @@ MaintenanceSchedulerTest::testPriorityNotClearedIfOperationNotStarted() _operationStarter->setShouldStartOperations(false); WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), waitMs); - CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri HIGH)\n"), + CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri HIGH)\n"), _priorityDb->toString()); } diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp index cca55a11b38..7adadd226d7 100644 --- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -254,7 +254,7 @@ PendingMessageTrackerTest::testSimple() CPPUNIT_ASSERT_CONTAIN( std::string( - "<b>Bucket(BucketSpace(0x0000000000000000), BucketId(0x40000000000004d2))</b>\n" + "<b>Bucket(BucketSpace(0x0000000000000001), BucketId(0x40000000000004d2))</b>\n" "<ul>\n" "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> " "Remove(BucketId(0x40000000000004d2), " @@ -341,14 +341,14 @@ PendingMessageTrackerTest::testMultipleMessages() CPPUNIT_ASSERT_CONTAIN( std::string( - "<b>Bucket(BucketSpace(0x0000000000000000), BucketId(0x40000000000004d2))</b>\n" + "<b>Bucket(BucketSpace(0x0000000000000001), BucketId(0x40000000000004d2))</b>\n" "<ul>\n" "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:0, timestamp 1000)</li>\n" "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:2, timestamp 1002)</li>\n" "<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:1, timestamp 1001)</li>\n" "<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:3, timestamp 1003)</li>\n" "</ul>\n" - "<b>Bucket(BucketSpace(0x0000000000000000), BucketId(0x40000000000011d7))</b>\n" + "<b>Bucket(BucketSpace(0x0000000000000001), BucketId(0x40000000000011d7))</b>\n" "<ul>\n" "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:0, timestamp 2000)</li>\n" "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:2, timestamp 2002)</li>\n" diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp index 7f54e163006..e621ef8645c 100644 --- a/storage/src/tests/distributor/putoperationtest.cpp +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -282,7 +282,7 @@ PutOperationTest::testNodeRemovedOnReply() CPPUNIT_ASSERT_EQUAL(std::string( "PutReply(doc:test:test, BucketId(0x0000000000000000), " "timestamp 100) ReturnCode(BUCKET_DELETED, " - "Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000008b13)) was deleted from nodes [0] " + "Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000008b13)) was deleted from nodes [0] " "after message was sent but before it was done. " "Sent to [1,0])"), _sender.getLastReply()); diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 66a2d3efa6c..394df6024fd 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -92,7 +92,7 @@ void SimpleMaintenanceScannerTest::testPrioritizeSingleBucket() { addBucketToDb(1); - std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"); auto scanResult = _scanner->scanNext(); CPPUNIT_ASSERT(!scanResult.isDone()); @@ -141,9 +141,9 @@ SimpleMaintenanceScannerTest::testPrioritizeMultipleBuckets() addBucketToDb(1); addBucketToDb(2); addBucketToDb(3); - std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" - "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri VERY_HIGH)\n" - "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"); CPPUNIT_ASSERT(scanEntireDatabase(3)); CPPUNIT_ASSERT_EQUAL(sortLines(expected), @@ -168,8 +168,8 @@ SimpleMaintenanceScannerTest::testReset() addBucketToDb(3); CPPUNIT_ASSERT(scanEntireDatabase(2)); - std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" - "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"); CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); addBucketToDb(2); @@ -179,9 +179,9 @@ SimpleMaintenanceScannerTest::testReset() _scanner->reset(); CPPUNIT_ASSERT(scanEntireDatabase(3)); - expected = "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" - "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri VERY_HIGH)\n" - "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"; + expected = "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000002)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"; CPPUNIT_ASSERT_EQUAL(sortLines(expected), sortLines(_priorityDb->toString())); } diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp index c3aebcafe06..c3290a8c0f6 100644 --- a/storage/src/tests/distributor/throttlingoperationstartertest.cpp +++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp @@ -70,7 +70,7 @@ ThrottlingOperationStarterTest::testOperationStartingIsForwardedToImplementation { CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); - CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 0\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri 0\n"), _starterImpl->toString()); } diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index 38fb0f6235a..95faf7e433e 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(storage_teststorageserver TEST bucketintegritycheckertest.cpp changedbucketownershiphandlertest.cpp communicationmanagertest.cpp + configurable_bucket_resolver_test.cpp documentapiconvertertest.cpp mergethrottlertest.cpp priorityconvertertest.cpp diff --git a/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp new file mode 100644 index 00000000000..3f121240065 --- /dev/null +++ b/storage/src/tests/storageserver/configurable_bucket_resolver_test.cpp @@ -0,0 +1,137 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/storage/storageserver/configurable_bucket_resolver.h> +#include <vespa/document/base/documentid.h> +#include <vespa/persistence/spi/fixed_bucket_spaces.h> +#include <cppunit/extensions/HelperMacros.h> + +namespace storage { + +using document::DocumentId; + +struct ConfigurableBucketResolverTest : CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ConfigurableBucketResolverTest); + CPPUNIT_TEST(bucket_space_from_name_is_defined_for_default_space); + CPPUNIT_TEST(bucket_space_from_name_is_defined_for_global_space); + CPPUNIT_TEST(bucket_space_from_name_throws_exception_for_unknown_space); + CPPUNIT_TEST(name_from_bucket_space_is_defined_for_default_space); + CPPUNIT_TEST(name_from_bucket_space_is_defined_for_global_space); + CPPUNIT_TEST(name_from_bucket_space_throws_exception_for_unknown_space); + CPPUNIT_TEST(known_bucket_space_is_resolved_from_document_id); + CPPUNIT_TEST(unknown_bucket_space_in_id_throws_exception); + CPPUNIT_TEST(can_create_resolver_from_bucket_space_config); + CPPUNIT_TEST_SUITE_END(); + + using BucketSpaceMapping = ConfigurableBucketResolver::BucketSpaceMapping; + + BucketSpaceMapping create_simple_mapping() { + return {{"foo", spi::FixedBucketSpaces::default_space()}, + {"bar", spi::FixedBucketSpaces::default_space()}, + {"baz", spi::FixedBucketSpaces::global_space()}}; + } + + ConfigurableBucketResolver create_empty_resolver() { + return ConfigurableBucketResolver({}); + } + + ConfigurableBucketResolver create_simple_resolver() { + return ConfigurableBucketResolver(create_simple_mapping()); + } + + void bucket_space_from_name_is_defined_for_default_space(); + void bucket_space_from_name_is_defined_for_global_space(); + void bucket_space_from_name_throws_exception_for_unknown_space(); + void name_from_bucket_space_is_defined_for_default_space(); + void name_from_bucket_space_is_defined_for_global_space(); + void name_from_bucket_space_throws_exception_for_unknown_space(); + void known_bucket_space_is_resolved_from_document_id(); + void unknown_bucket_space_in_id_throws_exception(); + void can_create_resolver_from_bucket_space_config(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ConfigurableBucketResolverTest); + +// TODO reduce overlap with FixedBucketSpacesTest +void ConfigurableBucketResolverTest::bucket_space_from_name_is_defined_for_default_space() { + auto space = create_empty_resolver().bucketSpaceFromName("default"); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(), space); +} + +void ConfigurableBucketResolverTest::bucket_space_from_name_is_defined_for_global_space() { + auto space = create_empty_resolver().bucketSpaceFromName("global"); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(), space); +} + +void ConfigurableBucketResolverTest::bucket_space_from_name_throws_exception_for_unknown_space() { + try { + create_empty_resolver().bucketSpaceFromName("bjarne"); + CPPUNIT_FAIL("Expected exception on unknown bucket space name"); + } catch (spi::UnknownBucketSpaceException& e) { + } +} + +void ConfigurableBucketResolverTest::name_from_bucket_space_is_defined_for_default_space() { + CPPUNIT_ASSERT_EQUAL(vespalib::string("default"), + create_empty_resolver().nameFromBucketSpace(spi::FixedBucketSpaces::default_space())); +} + +void ConfigurableBucketResolverTest::name_from_bucket_space_is_defined_for_global_space() { + CPPUNIT_ASSERT_EQUAL(vespalib::string("global"), + create_empty_resolver().nameFromBucketSpace(spi::FixedBucketSpaces::global_space())); +} + +void ConfigurableBucketResolverTest::name_from_bucket_space_throws_exception_for_unknown_space() { + try { + create_empty_resolver().nameFromBucketSpace(document::BucketSpace(1234)); + CPPUNIT_FAIL("Expected exception on unknown bucket space value"); + } catch (spi::UnknownBucketSpaceException& e) { + } +} + +void ConfigurableBucketResolverTest::known_bucket_space_is_resolved_from_document_id() { + auto resolver = create_simple_resolver(); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(), + resolver.bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace()); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(), + resolver.bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace()); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(), + resolver.bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace()); +} + +void ConfigurableBucketResolverTest::unknown_bucket_space_in_id_throws_exception() { + try { + create_simple_resolver().bucketFromId(DocumentId("id::bjarne::xyz")); + CPPUNIT_FAIL("Expected exception on unknown document type -> bucket space mapping"); + } catch (spi::UnknownBucketSpaceException& e) { + } +} + +using BucketSpacesConfigBuilder = vespa::config::content::core::BucketspacesConfigBuilder; + +namespace { + +BucketSpacesConfigBuilder::Documenttype make_doc_type(vespalib::stringref name, vespalib::stringref space) { + BucketSpacesConfigBuilder::Documenttype doc_type; + doc_type.name = name; + doc_type.bucketspace = space; + return doc_type; +} + +} + +void ConfigurableBucketResolverTest::can_create_resolver_from_bucket_space_config() { + BucketSpacesConfigBuilder builder; + builder.documenttype.emplace_back(make_doc_type("foo", "default")); + builder.documenttype.emplace_back(make_doc_type("bar", "global")); + builder.documenttype.emplace_back(make_doc_type("baz", "global")); + auto resolver = ConfigurableBucketResolver::from_config(builder); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::default_space(), + resolver->bucketFromId(DocumentId("id::foo::xyz")).getBucketSpace()); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(), + resolver->bucketFromId(DocumentId("id::bar::xyz")).getBucketSpace()); + CPPUNIT_ASSERT_EQUAL(spi::FixedBucketSpaces::global_space(), + resolver->bucketFromId(DocumentId("id::baz::xyz")).getBucketSpace()); +} + +} + diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index c0238922a91..4fb3a5a0b99 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_add_library(storage_storageserver changedbucketownershiphandler.cpp communicationmanager.cpp communicationmanagermetrics.cpp + configurable_bucket_resolver.cpp distributornode.cpp distributornodecontext.cpp documentapiconverter.cpp diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp new file mode 100644 index 00000000000..86c802a65cf --- /dev/null +++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.cpp @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/base/documentid.h> +#include <vespa/persistence/spi/fixed_bucket_spaces.h> +#include <vespa/vespalib/util/exceptions.h> +#include "configurable_bucket_resolver.h" + +namespace storage { + +document::Bucket ConfigurableBucketResolver::bucketFromId(const document::DocumentId& id) const { + auto iter = _type_to_space.find(id.getDocType()); + if (iter != _type_to_space.end()) { + return document::Bucket(iter->second, document::BucketId(0)); + } + throw spi::UnknownBucketSpaceException("Unknown bucket space mapping for document type '" + + id.getDocType() + "' in id: " + id.toString(), VESPA_STRLOC); +} + +document::BucketSpace ConfigurableBucketResolver::bucketSpaceFromName(const vespalib::string& name) const { + return spi::FixedBucketSpaces::from_string(name); +} + +vespalib::string ConfigurableBucketResolver::nameFromBucketSpace(const document::BucketSpace& space) const { + return spi::FixedBucketSpaces::to_string(space); +} + +std::shared_ptr<ConfigurableBucketResolver> ConfigurableBucketResolver::from_config( + const vespa::config::content::core::BucketspacesConfig& config) { + ConfigurableBucketResolver::BucketSpaceMapping type_to_space; + for (auto& mapping : config.documenttype) { + type_to_space.emplace(mapping.name, spi::FixedBucketSpaces::from_string(mapping.bucketspace)); + } + return std::make_shared<ConfigurableBucketResolver>(std::move(type_to_space)); +} + +} diff --git a/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h new file mode 100644 index 00000000000..acebd9777fb --- /dev/null +++ b/storage/src/vespa/storage/storageserver/configurable_bucket_resolver.h @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storage/config/config-bucketspaces.h> +#include <vespa/storage/common/bucket_resolver.h> +#include <vespa/vespalib/stllike/hash_fun.h> +#include <memory> +#include <unordered_map> + +namespace storage { + +/** + * Immutable implementation of BucketResolver which maintains an explicit + * mapping from document type to bucket space. + * + * If an unknown document type or bucket space is given as an argument, + * an spi::UnknownBucketSpaceException is thrown. + */ +class ConfigurableBucketResolver : public BucketResolver { +public: + using BucketSpaceMapping = std::unordered_map<vespalib::string, document::BucketSpace, vespalib::hash<vespalib::string>>; + const BucketSpaceMapping _type_to_space; +public: + explicit ConfigurableBucketResolver(BucketSpaceMapping type_to_space) + : _type_to_space(std::move(type_to_space)) + {} + + document::Bucket bucketFromId(const document::DocumentId&) const override; + document::BucketSpace bucketSpaceFromName(const vespalib::string& name) const override; + vespalib::string nameFromBucketSpace(const document::BucketSpace& space) const override; + + static std::shared_ptr<ConfigurableBucketResolver> from_config( + const vespa::config::content::core::BucketspacesConfig& config); +}; + +}
\ No newline at end of file |