From 28f5f6d0a22ceef90246a159d77cc8acc1005b19 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Thu, 12 Nov 2020 13:27:56 +0000 Subject: Enforce minimum bucked used bits at document metastore load time --- .../src/vespa/persistence/spi/bucket_limits.h | 17 ++++++++++++++ .../src/tests/proton/documentmetastore/.gitignore | 1 + .../documentmetastore/documentmetastore_test.cpp | 26 ++++++++++++++++++++++ .../proton/documentmetastore/documentmetastore.cpp | 3 ++- storage/src/vespa/storage/common/bucket_limits.h | 17 -------------- .../storage/config/distributorconfiguration.cpp | 4 ++-- 6 files changed, 48 insertions(+), 20 deletions(-) create mode 100644 persistence/src/vespa/persistence/spi/bucket_limits.h delete mode 100644 storage/src/vespa/storage/common/bucket_limits.h diff --git a/persistence/src/vespa/persistence/spi/bucket_limits.h b/persistence/src/vespa/persistence/spi/bucket_limits.h new file mode 100644 index 00000000000..65b2660eb23 --- /dev/null +++ b/persistence/src/vespa/persistence/spi/bucket_limits.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace storage::spi { + +/** + * Wrapper of constants that specify absolute lower and upper bounds for buckets + * that are to be processed on a node. These invariants must be maintained by + * split and join operations, as well as bucket creation. + */ +struct BucketLimits { + constexpr static uint8_t MinUsedBits = 8; + constexpr static uint8_t MaxUsedBits = 58; +}; + +} diff --git a/searchcore/src/tests/proton/documentmetastore/.gitignore b/searchcore/src/tests/proton/documentmetastore/.gitignore index 619f7adbc6c..d1d05764ae6 100644 --- a/searchcore/src/tests/proton/documentmetastore/.gitignore +++ b/searchcore/src/tests/proton/documentmetastore/.gitignore @@ -3,4 +3,5 @@ Makefile gidmapattribute_test /documentmetastore2.dat /documentmetastore3.dat +/documentmetastore4.dat searchcore_documentmetastore_test_app diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index b586b74ad00..326cfba97f4 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include +#include #include #include #include @@ -629,6 +630,31 @@ TEST(DocumentMetaStoreTest, gids_can_be_saved_and_loaded) } } +TEST(DocumentMetaStoreTest, bucket_used_bits_are_lbounded_at_load_time) +{ + DocumentMetaStore dms1(createBucketDB()); + dms1.constructFreeList(); + + constexpr uint32_t lid = 1; + GlobalId gid = createGid(lid); + BucketId bucketId(gid.convertToBucketId()); + bucketId.setUsedBits(storage::spi::BucketLimits::MinUsedBits - 1); + uint32_t added_lid = addGid(dms1, gid, bucketId, Timestamp(1000)); + ASSERT_EQ(added_lid, lid); + + TuneFileAttributes tuneFileAttributes; + DummyFileHeaderContext fileHeaderContext; + AttributeFileSaveTarget saveTarget(tuneFileAttributes, fileHeaderContext); + ASSERT_TRUE(dms1.save(saveTarget, "documentmetastore2")); + + DocumentMetaStore dms2(createBucketDB(), "documentmetastore2"); + ASSERT_TRUE(dms2.load()); + ASSERT_EQ(dms2.getNumDocs(), 2); // Incl. zero LID + + BucketId expected_bucket(storage::spi::BucketLimits::MinUsedBits, gid.convertToBucketId().getRawId()); + assertGid(gid, lid, dms2, expected_bucket, Timestamp(1000)); +} + TEST(DocumentMetaStore, stats_are_updated) { DocumentMetaStore dms(createBucketDB()); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index b3dfebfa9c0..2b36067a65c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -5,6 +5,7 @@ #include "operation_listener.h" #include "search_context.h" #include +#include #include #include #include @@ -102,7 +103,7 @@ public: uint8_t getNextBucketUsedBits() { - return _bucketUsedBitsReader.readHostOrder(); + return std::max(_bucketUsedBitsReader.readHostOrder(), storage::spi::BucketLimits::MinUsedBits); } Timestamp diff --git a/storage/src/vespa/storage/common/bucket_limits.h b/storage/src/vespa/storage/common/bucket_limits.h deleted file mode 100644 index f822c46c6e0..00000000000 --- a/storage/src/vespa/storage/common/bucket_limits.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include - -namespace storage { - -/** - * Wrapper of constants that specify absolute lower and upper bounds for buckets - * that are to be processed on a node. These invariants must be maintained by - * split and join operations, as well as bucket creation. - */ -struct BucketLimits { - constexpr static uint8_t MinUsedBits = 8; - constexpr static uint8_t MaxUsedBits = 58; -}; - -} diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 5a1a0ca04a0..cc04f1d554c 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -2,7 +2,7 @@ #include "distributorconfiguration.h" #include #include -#include +#include #include #include @@ -126,7 +126,7 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorDist _docCountSplitLimit = config.splitcount; _byteCountJoinLimit = config.joinsize; _docCountJoinLimit = config.joincount; - _minimalBucketSplit = std::max(config.minsplitcount, static_cast(BucketLimits::MinUsedBits)); + _minimalBucketSplit = std::max(config.minsplitcount, static_cast(spi::BucketLimits::MinUsedBits)); _maxNodesPerMerge = config.maximumNodesPerMerge; _max_consecutively_inhibited_maintenance_ticks = config.maxConsecutivelyInhibitedMaintenanceTicks; -- cgit v1.2.3