diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-11-08 21:54:03 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-11-09 09:48:52 +0000 |
commit | bcf3647d731302a7f6f55798d08680f133ab4f3d (patch) | |
tree | 6389c60b1fd26fbeeedd555fa958ec5299078793 /searchcore | |
parent | bb75c5975c5588b8e05e9bd62016d1dce0a6d654 (diff) |
Avoid cache miss and dependency stall when have virtual small objects for checksum aggregation.
Instead compute checksum inline with a simple switch.
Diffstat (limited to 'searchcore')
9 files changed, 141 insertions, 178 deletions
diff --git a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp index 998b1626aeb..aff5573e4e0 100644 --- a/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp +++ b/searchcore/src/tests/proton/bucketdb/bucketdb/bucketdb_test.cpp @@ -136,6 +136,10 @@ TEST_F("require that bucket checksum is a combination of sub db types", Fixture) EXPECT_EQUAL(zero, f.remove(TIME_3, SDT::REMOVED).getChecksum()); } +TEST("require that BucketState follows checksum type") { + EXPECT_EQUAL(48u, sizeof(BucketState)); +} + TEST_F("require that bucket is ready when not having docs in notready sub db", Fixture) { assertReady(true, f.get()); @@ -222,47 +226,47 @@ verifyChecksumCompliance(ChecksumAggregator::ChecksumType type) { GlobalId gid2("bbbbbbbbbbbb"); Timestamp t1(0); Timestamp t2(1); - auto ckaggr = ChecksumAggregator::create(type, BucketChecksum(0)); + BucketState::setChecksumType(type); + BucketState bs; - EXPECT_EQUAL(0u, ckaggr->getChecksum()); - ckaggr->addDoc(gid1, t1); - BucketChecksum afterAdd = ckaggr->getChecksum(); + EXPECT_EQUAL(0u, bs.getChecksum()); + bs.add(gid1, t1, 1, SubDbType::READY); + BucketChecksum afterAdd = bs.getChecksum(); EXPECT_NOT_EQUAL(0u, afterAdd); // add Changes checksum - ckaggr->removeDoc(gid1, t1); - EXPECT_EQUAL(0u, ckaggr->getChecksum()); // add/remove are symmetrical - ckaggr->addDoc(gid1, t2); - EXPECT_NOT_EQUAL(afterAdd, ckaggr->getChecksum()); // timestamp changes checksum - ckaggr->removeDoc(gid1, t2); - EXPECT_EQUAL(0u, ckaggr->getChecksum()); // add/remove are symmetrical - ckaggr->addDoc(gid2, t1); - EXPECT_NOT_EQUAL(afterAdd, ckaggr->getChecksum()); // gid changes checksum - ckaggr->removeDoc(gid2, t1); - EXPECT_EQUAL(0u, ckaggr->getChecksum()); // add/remove are symmetrical + bs.remove(gid1, t1, 1, SubDbType::READY); + EXPECT_EQUAL(0u, bs.getChecksum()); // add/remove are symmetrical + bs.add(gid1, t2, 1, SubDbType::READY); + EXPECT_NOT_EQUAL(afterAdd, bs.getChecksum()); // timestamp changes checksum + bs.remove(gid1, t2, 1, SubDbType::READY); + EXPECT_EQUAL(0u, bs.getChecksum()); // add/remove are symmetrical + bs.add(gid2, t1, 1, SubDbType::READY); + EXPECT_NOT_EQUAL(afterAdd, bs.getChecksum()); // gid changes checksum + bs.remove(gid2, t1, 1, SubDbType::READY); + EXPECT_EQUAL(0u, bs.getChecksum()); // add/remove are symmetrical { // Verify order does not matter, only current content. A,B == B,A - ckaggr->addDoc(gid1, t1); - BucketChecksum after1AddOfGid1 = ckaggr->getChecksum(); - ckaggr->addDoc(gid2, t2); - BucketChecksum after2Add1 = ckaggr->getChecksum(); - ckaggr->removeDoc(gid2, t2); - EXPECT_EQUAL(after1AddOfGid1, ckaggr->getChecksum()); - ckaggr->removeDoc(gid1, t1); - EXPECT_EQUAL(0u, ckaggr->getChecksum()); + bs.add(gid1, t1, 1, SubDbType::READY); + BucketChecksum after1AddOfGid1 = bs.getChecksum(); + bs.add(gid2, t2, 1, SubDbType::READY); + BucketChecksum after2Add1 = bs.getChecksum(); + bs.remove(gid2, t2, 1, SubDbType::READY); + EXPECT_EQUAL(after1AddOfGid1, bs.getChecksum()); + bs.remove(gid1, t1, 1, SubDbType::READY); + EXPECT_EQUAL(0u, bs.getChecksum()); - ckaggr->addDoc(gid2, t2); - EXPECT_NOT_EQUAL(after1AddOfGid1, ckaggr->getChecksum()); - ckaggr->addDoc(gid1, t1); - EXPECT_EQUAL(after2Add1, ckaggr->getChecksum()); - ckaggr->removeDoc(gid2, t2); - EXPECT_EQUAL(after1AddOfGid1, ckaggr->getChecksum()); - ckaggr->removeDoc(gid1, t1); - EXPECT_EQUAL(0u, ckaggr->getChecksum()); // add/remove are symmetrical + bs.add(gid2, t2, 1, SubDbType::READY); + EXPECT_NOT_EQUAL(after1AddOfGid1, bs.getChecksum()); + bs.add(gid1, t1, 1, SubDbType::READY); + EXPECT_EQUAL(after2Add1, bs.getChecksum()); + bs.remove(gid2, t2, 1, SubDbType::READY); + EXPECT_EQUAL(after1AddOfGid1, bs.getChecksum()); + bs.remove(gid1, t1, 1, SubDbType::READY); + EXPECT_EQUAL(0u, bs.getChecksum()); // add/remove are symmetrical } - ckaggr->addDoc(gid1, t1); // Add something so we can verify it does not change between releases. - - return ckaggr->getChecksum(); + bs.add(gid1, t1, 1, SubDbType::READY); // Add something so we can verify it does not change between releases. + return bs.getChecksum(); } TEST("test that legacy checksum complies") { diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 60d96e37e15..db0f26a9d55 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -759,7 +759,6 @@ TEST(DocumentMetaStoreTest, can_sort_gids) } } -template <typename ChecksumType> void requireThatBasicBucketInfoWorks() { @@ -773,8 +772,7 @@ requireThatBasicBucketInfoWorks() GlobalId gid = createGid(lid); Timestamp timestamp(UINT64_C(123456789) * lid); Timestamp oldTimestamp; - BucketId bucketId(minNumBits, - gid.convertToBucketId().getRawId()); + BucketId bucketId(minNumBits, gid.convertToBucketId().getRawId()); uint32_t addLid = addGid(dms, gid, bucketId, timestamp); EXPECT_EQ(lid, addLid); m[std::make_pair(bucketId, gid)] = timestamp; @@ -783,37 +781,35 @@ requireThatBasicBucketInfoWorks() GlobalId gid = createGid(lid); Timestamp timestamp(UINT64_C(14735) * lid); Timestamp oldTimestamp; - BucketId bucketId(minNumBits, - gid.convertToBucketId().getRawId()); + BucketId bucketId(minNumBits, gid.convertToBucketId().getRawId()); uint32_t addLid = addGid(dms, gid, bucketId, timestamp); EXPECT_EQ(lid, addLid); m[std::make_pair(bucketId, gid)] = timestamp; } for (uint32_t lid = 3; lid <= numLids; lid += 5) { GlobalId gid = createGid(lid); - BucketId bucketId(minNumBits, - gid.convertToBucketId().getRawId()); + BucketId bucketId(minNumBits, gid.convertToBucketId().getRawId()); EXPECT_TRUE(dms.remove(lid, 0u)); dms.removeComplete(lid); m.erase(std::make_pair(bucketId, gid)); } assert(!m.empty()); - ChecksumType cksum(BucketChecksum(0)); + BucketState cksum; BucketId prevBucket = m.begin()->first.first; uint32_t cnt = 0u; uint32_t maxcnt = 0u; BucketDBOwner::Guard bucketDB = dms.getBucketDB().takeGuard(); for (Map::const_iterator i = m.begin(), ie = m.end(); i != ie; ++i) { if (i->first.first == prevBucket) { - cksum.addDoc(i->first.second, i->second); + cksum.add(i->first.second, i->second, 1, SubDbType::READY); ++cnt; } else { BucketInfo bi = bucketDB->get(prevBucket); EXPECT_EQ(cnt, bi.getDocumentCount()); EXPECT_EQ(cksum.getChecksum(), bi.getChecksum()); prevBucket = i->first.first; - cksum = ChecksumType(BucketChecksum(0)); - cksum.addDoc(i->first.second, i->second); + cksum = BucketState(); + cksum.add(i->first.second, i->second, 1, SubDbType::READY); maxcnt = std::max(maxcnt, cnt); cnt = 1u; } @@ -828,9 +824,9 @@ requireThatBasicBucketInfoWorks() TEST(DocumentMetaStoreTest, basic_bucket_info_works) { BucketState::setChecksumType(BucketState::ChecksumType::LEGACY); - requireThatBasicBucketInfoWorks<LegacyChecksumAggregator>(); + requireThatBasicBucketInfoWorks(); BucketState::setChecksumType(BucketState::ChecksumType::XXHASH64); - requireThatBasicBucketInfoWorks<XXH64ChecksumAggregator>(); + requireThatBasicBucketInfoWorks(); } TEST(DocumentMetaStoreTest, can_retrieve_list_of_lids_from_bucket_id) diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt index 375515938b3..6619f9e419d 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_library(searchcore_bucketdb STATIC bucketdbhandler.cpp bucketsessionbase.cpp bucketstate.cpp - checksumaggregator.cpp checksumaggregators.cpp joinbucketssession.cpp splitbucketsession.cpp diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp index cea30680faf..ea4c197ef02 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp @@ -18,11 +18,6 @@ toIdx(SubDbType subDbType) { BucketState::ChecksumType BucketState::_checksumType = BucketState::ChecksumType::LEGACY; -std::unique_ptr<ChecksumAggregator> -BucketState::createChecksum(BucketChecksum seed) { - return ChecksumAggregator::create(_checksumType, seed); -} - void BucketState::setChecksumType(ChecksumType type) { _checksumType = type; @@ -34,11 +29,12 @@ BucketState::BucketState(BucketState && rhs) noexcept = default; BucketState & BucketState::operator=(BucketState && rhs) noexcept = default; BucketState::BucketState() - : _docCount(), + : _ch(), _docSizes(), - _checksum(createChecksum(BucketChecksum(0))), + _docCount(), _active(false) { + memset(&_ch, 0, sizeof(_ch)); for (uint32_t i = 0; i < COUNTS; ++i) { _docCount[i] = 0; _docSizes[i] = 0; @@ -46,8 +42,25 @@ BucketState::BucketState() } BucketState::BucketChecksum +BucketState::getChecksum() const { + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + return LegacyChecksumAggregator::get(_ch._legacy); + case ChecksumAggregator::ChecksumType::XXHASH64: + return XXH64ChecksumAggregator::get(_ch._xxh64); + } + abort(); +} + +BucketState::BucketChecksum BucketState::addChecksum(BucketChecksum a, BucketChecksum b) { - return createChecksum(a)->addChecksum(*createChecksum(b)).getChecksum(); + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + return LegacyChecksumAggregator::get(LegacyChecksumAggregator::add(b, a)); + case ChecksumAggregator::ChecksumType::XXHASH64: + return XXH64ChecksumAggregator::get(XXH64ChecksumAggregator::update(b, a)); + } + abort(); } void @@ -55,7 +68,14 @@ BucketState::add(const GlobalId &gid, const Timestamp ×tamp, uint32_t docSi { assert(subDbType < SubDbType::COUNT); if (subDbType != SubDbType::REMOVED) { - _checksum->addDoc(gid, timestamp); + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + _ch._legacy = LegacyChecksumAggregator::addDoc(gid, timestamp, _ch._legacy); + break; + case ChecksumAggregator::ChecksumType::XXHASH64: + _ch._xxh64 = XXH64ChecksumAggregator::update(gid, timestamp, _ch._xxh64); + break; + } } uint32_t subDbTypeIdx = toIdx(subDbType); ++_docCount[subDbTypeIdx]; @@ -70,7 +90,14 @@ BucketState::remove(const GlobalId &gid, const Timestamp ×tamp, uint32_t do assert(_docCount[subDbTypeIdx] > 0); assert(_docSizes[subDbTypeIdx] >= docSize); if (subDbType != SubDbType::REMOVED) { - _checksum->removeDoc(gid, timestamp); + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + _ch._legacy = LegacyChecksumAggregator::removeDoc(gid, timestamp, _ch._legacy); + break; + case ChecksumAggregator::ChecksumType::XXHASH64: + _ch._xxh64 = XXH64ChecksumAggregator::update(gid, timestamp, _ch._xxh64); + break; + } } --_docCount[subDbTypeIdx]; _docSizes[subDbTypeIdx] -= docSize; @@ -87,8 +114,16 @@ BucketState::modify(const GlobalId &gid, assert(_docCount[subDbTypeIdx] > 0); assert(_docSizes[subDbTypeIdx] >= oldDocSize); if (subDbType != SubDbType::REMOVED) { - _checksum->removeDoc(gid, oldTimestamp); - _checksum->addDoc(gid, newTimestamp); + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + _ch._legacy = LegacyChecksumAggregator::removeDoc(gid, oldTimestamp, _ch._legacy); + _ch._legacy = LegacyChecksumAggregator::addDoc(gid, newTimestamp, _ch._legacy); + break; + case ChecksumAggregator::ChecksumType::XXHASH64: + _ch._xxh64 = XXH64ChecksumAggregator::update(gid, oldTimestamp, _ch._xxh64); + _ch._xxh64 = XXH64ChecksumAggregator::update(gid, newTimestamp, _ch._xxh64); + break; + } } _docSizes[subDbTypeIdx] = _docSizes[subDbTypeIdx] + newDocSize - oldDocSize; } @@ -99,7 +134,8 @@ BucketState::empty() const if (getReadyCount() != 0 || getRemovedCount() != 0 || getNotReadyCount() != 0) return false; - assert(_checksum->empty()); + assert(_ch._xxh64 == 0); + assert(_ch._legacy == 0); for (uint32_t i = 0; i < COUNTS; ++i) { assert(_docSizes[i] == 0); } @@ -115,7 +151,15 @@ BucketState::operator+=(const BucketState &rhs) for (uint32_t i = 0; i < COUNTS; ++i) { _docSizes[i] += rhs._docSizes[i]; } - _checksum->addChecksum(*rhs._checksum); + + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + _ch._legacy = LegacyChecksumAggregator::add(rhs._ch._legacy, _ch._legacy); + break; + case ChecksumAggregator::ChecksumType::XXHASH64: + _ch._xxh64 = XXH64ChecksumAggregator::update(rhs._ch._xxh64, _ch._xxh64); + break; + } return *this; } @@ -134,7 +178,14 @@ BucketState::operator-=(const BucketState &rhs) for (uint32_t i = 0; i < COUNTS; ++i) { _docSizes[i] -= rhs._docSizes[i]; } - _checksum->removeChecksum(*rhs._checksum); + switch (_checksumType) { + case ChecksumAggregator::ChecksumType::LEGACY: + _ch._legacy = LegacyChecksumAggregator::remove(rhs._ch._legacy, _ch._legacy); + break; + case ChecksumAggregator::ChecksumType::XXHASH64: + _ch._xxh64 = XXH64ChecksumAggregator::update(rhs._ch._xxh64, _ch._xxh64); + break; + } return *this; } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h index 8c390b288ae..7f16500fad0 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.h @@ -26,9 +26,9 @@ private: static constexpr uint32_t REMOVED = static_cast<uint32_t>(SubDbType::REMOVED); static constexpr uint32_t NOTREADY = static_cast<uint32_t>(SubDbType::NOTREADY); static constexpr uint32_t COUNTS = static_cast<uint32_t>(SubDbType::COUNT); - uint32_t _docCount[COUNTS]; + union { uint32_t _legacy; uint64_t _xxh64;} _ch; size_t _docSizes[COUNTS]; - vespalib::CloneablePtr<ChecksumAggregator> _checksum; + uint32_t _docCount[COUNTS]; bool _active; static ChecksumType _checksumType; @@ -67,7 +67,7 @@ public: size_t getNotReadyDocSizes() const { return _docSizes[NOTREADY]; } uint32_t getDocumentCount() const { return getReadyCount() + getNotReadyCount(); } uint32_t getEntryCount() const { return getDocumentCount() + getRemovedCount(); } - BucketChecksum getChecksum() const { return _checksum->getChecksum(); } + BucketChecksum getChecksum() const; bool empty() const; BucketState &operator+=(const BucketState &rhs); BucketState &operator-=(const BucketState &rhs); diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.cpp deleted file mode 100644 index 795d88ab106..00000000000 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "checksumaggregators.h" - -namespace proton::bucketdb { - -std::unique_ptr<ChecksumAggregator> -ChecksumAggregator::create(ChecksumType type, BucketChecksum seed) { - switch (type) { - case ChecksumType::LEGACY: - return std::make_unique<LegacyChecksumAggregator>(seed); - case ChecksumType::XXHASH64: - return std::make_unique<XXH64ChecksumAggregator>(seed); - } - abort(); -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.h b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.h index 421cca5c6b5..b206f11f02f 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregator.h @@ -16,15 +16,6 @@ public: using GlobalId = document::GlobalId; using Timestamp = storage::spi::Timestamp; using BucketChecksum = storage::spi::BucketChecksum; - virtual ~ChecksumAggregator() = default; - virtual ChecksumAggregator & addDoc(const GlobalId &gid, const Timestamp ×tamp) = 0; - virtual ChecksumAggregator & removeDoc(const GlobalId &gid, const Timestamp ×tamp) = 0; - virtual ChecksumAggregator & addChecksum(const ChecksumAggregator & rhs) = 0; - virtual ChecksumAggregator & removeChecksum(const ChecksumAggregator & rhs) = 0; - virtual BucketChecksum getChecksum() const = 0; - virtual bool empty() const = 0;; - virtual ChecksumAggregator * clone() const = 0; - static std::unique_ptr<ChecksumAggregator> create(ChecksumType type, BucketChecksum seed); }; } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.cpp index a118f416b6b..9246f94ca83 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.cpp @@ -33,79 +33,30 @@ calcChecksum(const GlobalId &gid, const Timestamp ×tamp) return gidChecksum(gid) + timestampChecksum(timestamp); } +uint64_t +compute(const GlobalId &gid, const Timestamp ×tamp) { + char buffer[20]; + memcpy(&buffer[0], gid.get(), GlobalId::LENGTH); + uint64_t tmp = timestamp.getValue(); + memcpy(&buffer[GlobalId::LENGTH], &tmp, sizeof(tmp)); + return XXH64(buffer, sizeof(buffer), 0); } -LegacyChecksumAggregator * -LegacyChecksumAggregator::clone() const { - return new LegacyChecksumAggregator(*this); -} -LegacyChecksumAggregator & -LegacyChecksumAggregator::addDoc(const GlobalId &gid, const Timestamp ×tamp) { - _checksum += calcChecksum(gid, timestamp); - return *this; } -LegacyChecksumAggregator & -LegacyChecksumAggregator::removeDoc(const GlobalId &gid, const Timestamp ×tamp) { - _checksum -= calcChecksum(gid, timestamp); - return *this; -} -LegacyChecksumAggregator & -LegacyChecksumAggregator::addChecksum(const ChecksumAggregator & rhs) { - _checksum += dynamic_cast<const LegacyChecksumAggregator &>(rhs)._checksum; - return *this; -} -LegacyChecksumAggregator & -LegacyChecksumAggregator::removeChecksum(const ChecksumAggregator & rhs) { - _checksum -= dynamic_cast<const LegacyChecksumAggregator &>(rhs)._checksum; - return *this; -} -BucketChecksum -LegacyChecksumAggregator::getChecksum() const { - return BucketChecksum(_checksum); -} -bool -LegacyChecksumAggregator::empty() const { return _checksum == 0; } - - -XXH64ChecksumAggregator * -XXH64ChecksumAggregator::clone() const { - return new XXH64ChecksumAggregator(*this); -} -XXH64ChecksumAggregator & -XXH64ChecksumAggregator::addDoc(const GlobalId &gid, const Timestamp ×tamp) { - _checksum ^= compute(gid, timestamp); - return *this; -} -XXH64ChecksumAggregator & -XXH64ChecksumAggregator::removeDoc(const GlobalId &gid, const Timestamp ×tamp) { - _checksum ^= compute(gid, timestamp); - return *this; -} -XXH64ChecksumAggregator & -XXH64ChecksumAggregator::addChecksum(const ChecksumAggregator & rhs) { - _checksum ^= dynamic_cast<const XXH64ChecksumAggregator &>(rhs)._checksum; - return *this; -} -XXH64ChecksumAggregator & -XXH64ChecksumAggregator::removeChecksum(const ChecksumAggregator & rhs) { - _checksum ^= dynamic_cast<const XXH64ChecksumAggregator &>(rhs)._checksum; - return *this; +uint32_t +LegacyChecksumAggregator::addDoc(const GlobalId &gid, const Timestamp ×tamp, uint32_t checkSum) { + return add(calcChecksum(gid, timestamp), checkSum); } -BucketChecksum -XXH64ChecksumAggregator::getChecksum() const { - return BucketChecksum((_checksum >> 32) ^ (_checksum & 0xffffffffL)); + +uint32_t +LegacyChecksumAggregator::removeDoc(const GlobalId &gid, const Timestamp ×tamp, uint32_t checkSum) { + return remove(calcChecksum(gid, timestamp), checkSum); } -bool -XXH64ChecksumAggregator::empty() const { return _checksum == 0; } uint64_t -XXH64ChecksumAggregator::compute(const GlobalId &gid, const Timestamp ×tamp) { - char buffer[20]; - memcpy(&buffer[0], gid.get(), GlobalId::LENGTH); - uint64_t tmp = timestamp.getValue(); - memcpy(&buffer[GlobalId::LENGTH], &tmp, sizeof(tmp)); - return XXH64(buffer, sizeof(buffer), 0); +XXH64ChecksumAggregator::update(const GlobalId &gid, const Timestamp ×tamp, uint64_t checkSum) { + return update(compute(gid, timestamp), checkSum); } } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.h b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.h index 49762ad107f..1c0c2bc8f7a 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/checksumaggregators.h @@ -9,16 +9,11 @@ namespace proton::bucketdb { **/ class LegacyChecksumAggregator : public ChecksumAggregator { public: - explicit LegacyChecksumAggregator(BucketChecksum seed) : _checksum(seed) { } - LegacyChecksumAggregator * clone() const override; - LegacyChecksumAggregator & addDoc(const GlobalId &gid, const Timestamp ×tamp) override; - LegacyChecksumAggregator & removeDoc(const GlobalId &gid, const Timestamp ×tamp) override; - LegacyChecksumAggregator & addChecksum(const ChecksumAggregator & rhs) override; - LegacyChecksumAggregator & removeChecksum(const ChecksumAggregator & rhs) override; - BucketChecksum getChecksum() const override; - bool empty() const override; -private: - uint32_t _checksum; + static uint32_t addDoc(const GlobalId &gid, const Timestamp ×tamp, uint32_t checkSum); + static uint32_t removeDoc(const GlobalId &gid, const Timestamp ×tamp, uint32_t checkSum); + static uint32_t add(uint32_t checksum, uint32_t aggr) { return aggr + checksum; } + static uint32_t remove(uint32_t checksum, uint32_t aggr) { return aggr - checksum; } + static BucketChecksum get(uint32_t checkSum) { return BucketChecksum(checkSum); } }; /** @@ -26,17 +21,11 @@ private: **/ class XXH64ChecksumAggregator : public ChecksumAggregator { public: - explicit XXH64ChecksumAggregator(BucketChecksum seed) : _checksum(seed) { } - XXH64ChecksumAggregator * clone() const override; - XXH64ChecksumAggregator & addDoc(const GlobalId &gid, const Timestamp ×tamp) override; - XXH64ChecksumAggregator & removeDoc(const GlobalId &gid, const Timestamp ×tamp) override; - XXH64ChecksumAggregator & addChecksum(const ChecksumAggregator & rhs) override; - XXH64ChecksumAggregator & removeChecksum(const ChecksumAggregator & rhs) override; - BucketChecksum getChecksum() const override; - bool empty() const override; -private: - static uint64_t compute(const GlobalId &gid, const Timestamp ×tamp); - uint64_t _checksum; + static uint64_t update(const GlobalId &gid, const Timestamp ×tamp, uint64_t checkSum); + static uint64_t update(uint64_t a, uint64_t b) { return a ^ b; } + static BucketChecksum get(uint64_t checkSum) { + return BucketChecksum((checkSum >> 32) ^ (checkSum & 0xffffffffL)); + } }; } |