diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-10-14 10:04:20 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-10-14 10:04:20 +0000 |
commit | 760768ad9a5f376275b06ed38a516e80cba1aa65 (patch) | |
tree | 0e862282256eb77183a40f545fcf8faed03d8470 /searchcore | |
parent | 5c2bff00c1b5e0aea4bee32f43dd9ecda8b2ff7f (diff) |
Add serialisation for sorting to documentmetastore.
Diffstat (limited to 'searchcore')
5 files changed, 78 insertions, 28 deletions
diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index f25446d918f..6f4ebe2b537 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -1965,6 +1965,38 @@ TEST(DocumentMetaStoreTest, multiple_lids_can_be_removed_with_removeBatch) dms.removes_complete({1, 3}); } +TEST(DocumentMetaStoreTest, serialize_for_sort) +{ + DocumentMetaStore dms(createBucketDB()); + dms.constructFreeList(); + addLid(dms, 1); + addLid(dms, 2); + assertLidGidFound(1, dms); + assertLidGidFound(2, dms); + + constexpr size_t SZ = document::GlobalId::LENGTH; + EXPECT_EQ(12u, SZ); + EXPECT_EQ(SZ, dms.getFixedWidth()); + uint8_t asc_dest[SZ]; + EXPECT_EQ(0, dms.serializeForAscendingSort(3, asc_dest, sizeof(asc_dest), nullptr)); + EXPECT_EQ(-1, dms.serializeForAscendingSort(1, asc_dest, sizeof(asc_dest) - 1, nullptr)); + document::GlobalId gid; + + EXPECT_EQ(SZ, dms.serializeForAscendingSort(1, asc_dest, sizeof(asc_dest), nullptr)); + EXPECT_TRUE(dms.getGid(1, gid)); + EXPECT_EQ(0, memcmp(asc_dest, gid.get(), SZ)); + + EXPECT_EQ(SZ, dms.serializeForAscendingSort(2, asc_dest, sizeof(asc_dest), nullptr)); + EXPECT_TRUE(dms.getGid(2, gid)); + EXPECT_EQ(0, memcmp(asc_dest, gid.get(), SZ)); + + uint8_t desc_dest[SZ]; + EXPECT_EQ(SZ, dms.serializeForDescendingSort(2, desc_dest, sizeof(desc_dest), nullptr)); + for (size_t i(0); i < SZ; i++) { + EXPECT_EQ(0xff - asc_dest[i], desc_dest[i]); + } +} + class MockOperationListener : public documentmetastore::OperationListener { public: size_t remove_batch_cnt; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index ead9f142508..77452b60f21 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -72,17 +72,8 @@ private: uint32_t _version; public: - Reader(std::unique_ptr<FastOS_FileInterface> datFile) - : _datFile(std::move(datFile)), - _lidReader(&_datFile.file()), - _gidReader(&_datFile.file()), - _bucketUsedBitsReader(&_datFile.file()), - _timestampReader(&_datFile.file()), - _docIdLimit(0) - { - _docIdLimit = _datFile.header().getTag(DOCID_LIMIT).asInteger(); - _version = _datFile.header().getTag(VERSION).asInteger(); - } + explicit Reader(std::unique_ptr<FastOS_FileInterface> datFile); + ~Reader(); uint32_t getDocIdLimit() const { return _docIdLimit; } @@ -126,6 +117,19 @@ public: } }; +Reader::Reader(std::unique_ptr<FastOS_FileInterface> datFile) + : _datFile(std::move(datFile)), + _lidReader(&_datFile.file()), + _gidReader(&_datFile.file()), + _bucketUsedBitsReader(&_datFile.file()), + _timestampReader(&_datFile.file()), + _docIdLimit(0) +{ + _docIdLimit = _datFile.header().getTag(DOCID_LIMIT).asInteger(); + _version = _datFile.header().getTag(VERSION).asInteger(); +} +Reader::~Reader() = default; + } namespace { @@ -134,12 +138,12 @@ class ShrinkBlockHeld : public GenerationHeldBase DocumentMetaStore &_dms; public: - ShrinkBlockHeld(DocumentMetaStore &dms) + explicit ShrinkBlockHeld(DocumentMetaStore &dms) : GenerationHeldBase(0), _dms(dms) { } - ~ShrinkBlockHeld() { + ~ShrinkBlockHeld() override { _dms.unblockShrinkLidSpace(); } }; @@ -748,12 +752,12 @@ DocumentMetaStore::getMetaData(const GlobalId &gid) const { DocId lid = 0; if (!getLid(gid, lid) || !validLid(lid)) { - return search::DocumentMetaData(); + return {}; } const RawDocumentMetaData &raw = getRawMetaData(lid); Timestamp timestamp(raw.getTimestamp()); std::atomic_thread_fence(std::memory_order_acquire); - return search::DocumentMetaData(lid, timestamp, raw.getBucketId(), raw.getGid(), _subDbType == SubDbType::REMOVED); + return {lid, timestamp, raw.getBucketId(), raw.getGid(), _subDbType == SubDbType::REMOVED}; } void @@ -779,14 +783,7 @@ DocumentMetaStore::getMetaData(const BucketId &bucketId, LidUsageStats DocumentMetaStore::getLidUsageStats() const { - uint32_t docIdLimit = getCommittedDocIdLimit(); - uint32_t numDocs = getNumUsedLids(); - uint32_t lowestFreeLid = _lidAlloc.getLowestFreeLid(); - uint32_t highestUsedLid = _lidAlloc.getHighestUsedLid(); - return LidUsageStats(docIdLimit, - numDocs, - lowestFreeLid, - highestUsedLid); + return {getCommittedDocIdLimit(), getNumUsedLids(), _lidAlloc.getLowestFreeLid(), _lidAlloc.getHighestUsedLid()}; } Blueprint::UP @@ -1064,7 +1061,7 @@ DocumentMetaStore::getBucketOf(const vespalib::GenerationHandler::Guard &, uint3 if (__builtin_expect(validLidFast(lid, getCommittedDocIdLimit()), true)) { return getRawMetaData(lid).getBucketId(); } - return BucketId(); + return {}; } vespalib::GenerationHandler::Guard @@ -1094,6 +1091,26 @@ DocumentMetaStore::foreach(const search::IGidToLidMapperVisitor &visitor) const { visitor.visit(getRawMetaData(key.get_lid()).getGid(), key.get_lid()); }); } +long +DocumentMetaStore::onSerializeForAscendingSort(DocId lid, void * serTo, long available, const search::common::BlobConverter *) const { + if ( ! validLid(lid)) return 0; + if (available < document::GlobalId::LENGTH) return -1; + memcpy(serTo, getRawMetaData(lid).getGid().get(), document::GlobalId::LENGTH); + return document::GlobalId::LENGTH; +} + +long +DocumentMetaStore::onSerializeForDescendingSort(DocId lid, void * serTo, long available, const search::common::BlobConverter *) const { + if ( ! validLid(lid)) return 0; + if (available < document::GlobalId::LENGTH) return -1; + const auto * src(static_cast<const uint8_t *>(getRawMetaData(lid).getGid().get())); + auto * dst = static_cast<uint8_t *>(serTo); + for (size_t i(0); i < document::GlobalId::LENGTH; ++i) { + dst[i] = 0xff - src[i]; + } + return document::GlobalId::LENGTH; +} + } // namespace proton namespace vespalib::btree { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 401b210f6e0..ba944e3493d 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -272,6 +272,8 @@ public: uint32_t getVersion() const override; void setTrackDocumentSizes(bool trackDocumentSizes) { _trackDocumentSizes = trackDocumentSizes; } void foreach(const search::IGidToLidMapperVisitor &visitor) const override; + long onSerializeForAscendingSort(DocId, void *, long, const search::common::BlobConverter *) const override; + long onSerializeForDescendingSort(DocId, void *, long, const search::common::BlobConverter *) const override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp index 57c3159645b..5deeaf924ae 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp @@ -7,14 +7,14 @@ namespace proton { namespace { -const vespalib::string _G_documentMetaStoreName("[documentmetastore]"); +const vespalib::string documentMetaStoreName("[documentmetastore]"); } const vespalib::string & DocumentMetaStoreAttribute::getFixedName() { - return _G_documentMetaStoreName; + return documentMetaStoreName; } DocumentMetaStoreAttribute::DocumentMetaStoreAttribute(const vespalib::string &name) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h index 1a5f9c0077e..36408a01c17 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h @@ -14,12 +14,11 @@ namespace proton { class DocumentMetaStoreAttribute : public search::NotImplementedAttribute { public: - DocumentMetaStoreAttribute(const vespalib::string &name=getFixedName()); + explicit DocumentMetaStoreAttribute(const vespalib::string &name); ~DocumentMetaStoreAttribute() override; static const vespalib::string &getFixedName(); - // Implements IAttributeVector size_t getFixedWidth() const override { return document::GlobalId::LENGTH; } |