Merge pull request #24453 from vespa-engine/balder/use-gid-for-docid-sorting

Balder/use gid for docid sorting

6 files changed, 83 insertions, 33 deletions

diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
index f25446d918f..4668b8c65ab 100644
--- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
@@ -1965,6 +1965,38 @@ TEST(DocumentMetaStoreTest, multiple_lids_can_be_removed_with_removeBatch)
     dms.removes_complete({1, 3});
 }
 
+TEST(DocumentMetaStoreTest, serialize_for_sort)
+{
+    DocumentMetaStore dms(createBucketDB());
+    dms.constructFreeList();
+    addLid(dms, 1);
+    addLid(dms, 2);
+    assertLidGidFound(1, dms);
+    assertLidGidFound(2, dms);
+
+    constexpr size_t SZ = document::GlobalId::LENGTH;
+    EXPECT_EQ(12u, SZ);
+    EXPECT_EQ(SZ, dms.getFixedWidth());
+    uint8_t asc_dest[SZ];
+    EXPECT_EQ(0, dms.serializeForAscendingSort(3, asc_dest, sizeof(asc_dest), nullptr));
+    EXPECT_EQ(-1, dms.serializeForAscendingSort(1, asc_dest, sizeof(asc_dest) - 1, nullptr));
+    document::GlobalId gid;
+
+    EXPECT_EQ(SZ, dms.serializeForAscendingSort(1, asc_dest, sizeof(asc_dest), nullptr));
+    EXPECT_TRUE(dms.getGid(1, gid));
+    EXPECT_EQ(0, memcmp(asc_dest, gid.get(), SZ));
+
+    EXPECT_EQ(SZ, dms.serializeForAscendingSort(2, asc_dest, sizeof(asc_dest), nullptr));
+    EXPECT_TRUE(dms.getGid(2, gid));
+    EXPECT_EQ(0, memcmp(asc_dest, gid.get(), SZ));
+
+    uint8_t desc_dest[SZ];
+    EXPECT_EQ(SZ, dms.serializeForDescendingSort(2, desc_dest, sizeof(desc_dest), nullptr));
+    for (size_t i(0); i < SZ; i++) {
+        EXPECT_EQ(0xff - asc_dest[i], desc_dest[i]);
+    }
+}
+
 class MockOperationListener : public documentmetastore::OperationListener {
 public:
     size_t remove_batch_cnt;
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
index ead9f142508..77452b60f21 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
@@ -72,17 +72,8 @@ private:
     uint32_t _version;
 
 public:
-    Reader(std::unique_ptr<FastOS_FileInterface> datFile)
-        : _datFile(std::move(datFile)),
-          _lidReader(&_datFile.file()),
-          _gidReader(&_datFile.file()),
-          _bucketUsedBitsReader(&_datFile.file()),
-          _timestampReader(&_datFile.file()),
-          _docIdLimit(0)
-    {
-        _docIdLimit = _datFile.header().getTag(DOCID_LIMIT).asInteger();
-        _version = _datFile.header().getTag(VERSION).asInteger();
-    }
+    explicit Reader(std::unique_ptr<FastOS_FileInterface> datFile);
+    ~Reader();
 
     uint32_t getDocIdLimit() const { return _docIdLimit; }
 
@@ -126,6 +117,19 @@ public:
     }
 };
 
+Reader::Reader(std::unique_ptr<FastOS_FileInterface> datFile)
+    : _datFile(std::move(datFile)),
+      _lidReader(&_datFile.file()),
+      _gidReader(&_datFile.file()),
+      _bucketUsedBitsReader(&_datFile.file()),
+      _timestampReader(&_datFile.file()),
+      _docIdLimit(0)
+{
+    _docIdLimit = _datFile.header().getTag(DOCID_LIMIT).asInteger();
+    _version = _datFile.header().getTag(VERSION).asInteger();
+}
+Reader::~Reader() = default;
+
 }
 
 namespace {
@@ -134,12 +138,12 @@ class ShrinkBlockHeld : public GenerationHeldBase
     DocumentMetaStore &_dms;
 
 public:
-    ShrinkBlockHeld(DocumentMetaStore &dms)
+    explicit ShrinkBlockHeld(DocumentMetaStore &dms)
         : GenerationHeldBase(0),
           _dms(dms)
     { }
 
-    ~ShrinkBlockHeld() {
+    ~ShrinkBlockHeld() override {
         _dms.unblockShrinkLidSpace();
     }
 };
@@ -748,12 +752,12 @@ DocumentMetaStore::getMetaData(const GlobalId &gid) const
 {
     DocId lid = 0;
     if (!getLid(gid, lid) || !validLid(lid)) {
-        return search::DocumentMetaData();
+        return {};
     }
     const RawDocumentMetaData &raw = getRawMetaData(lid);
     Timestamp timestamp(raw.getTimestamp());
     std::atomic_thread_fence(std::memory_order_acquire);
-    return search::DocumentMetaData(lid, timestamp, raw.getBucketId(), raw.getGid(), _subDbType == SubDbType::REMOVED);
+    return {lid, timestamp, raw.getBucketId(), raw.getGid(), _subDbType == SubDbType::REMOVED};
 }
 
 void
@@ -779,14 +783,7 @@ DocumentMetaStore::getMetaData(const BucketId &bucketId,
 LidUsageStats
 DocumentMetaStore::getLidUsageStats() const
 {
-    uint32_t docIdLimit = getCommittedDocIdLimit();
-    uint32_t numDocs = getNumUsedLids();
-    uint32_t lowestFreeLid = _lidAlloc.getLowestFreeLid();
-    uint32_t highestUsedLid = _lidAlloc.getHighestUsedLid();
-    return LidUsageStats(docIdLimit,
-                         numDocs,
-                         lowestFreeLid,
-                         highestUsedLid);
+    return {getCommittedDocIdLimit(), getNumUsedLids(), _lidAlloc.getLowestFreeLid(), _lidAlloc.getHighestUsedLid()};
 }
 
 Blueprint::UP
@@ -1064,7 +1061,7 @@ DocumentMetaStore::getBucketOf(const vespalib::GenerationHandler::Guard &, uint3
     if (__builtin_expect(validLidFast(lid, getCommittedDocIdLimit()), true)) {
         return getRawMetaData(lid).getBucketId();
     }
-    return BucketId();
+    return {};
 }
 
 vespalib::GenerationHandler::Guard
@@ -1094,6 +1091,26 @@ DocumentMetaStore::foreach(const search::IGidToLidMapperVisitor &visitor) const
                               { visitor.visit(getRawMetaData(key.get_lid()).getGid(), key.get_lid()); });
 }
 
+long
+DocumentMetaStore::onSerializeForAscendingSort(DocId lid, void * serTo, long available, const search::common::BlobConverter *) const {
+    if ( ! validLid(lid)) return 0;
+    if (available < document::GlobalId::LENGTH) return -1;
+    memcpy(serTo, getRawMetaData(lid).getGid().get(), document::GlobalId::LENGTH);
+    return document::GlobalId::LENGTH;
+}
+
+long
+DocumentMetaStore::onSerializeForDescendingSort(DocId lid, void * serTo, long available, const search::common::BlobConverter *) const {
+    if ( ! validLid(lid)) return 0;
+    if (available < document::GlobalId::LENGTH) return -1;
+    const auto * src(static_cast<const uint8_t *>(getRawMetaData(lid).getGid().get()));
+    auto * dst = static_cast<uint8_t *>(serTo);
+    for (size_t i(0); i < document::GlobalId::LENGTH; ++i) {
+        dst[i] = 0xff - src[i];
+    }
+    return document::GlobalId::LENGTH;
+}
+
 }  // namespace proton
 
 namespace vespalib::btree {
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
index 401b210f6e0..ba944e3493d 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
@@ -272,6 +272,8 @@ public:
     uint32_t getVersion() const override;
     void setTrackDocumentSizes(bool trackDocumentSizes) { _trackDocumentSizes = trackDocumentSizes; }
     void foreach(const search::IGidToLidMapperVisitor &visitor) const override;
+    long onSerializeForAscendingSort(DocId, void *, long, const search::common::BlobConverter *) const override;
+    long onSerializeForDescendingSort(DocId, void *, long, const search::common::BlobConverter *) const override;
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp
index 57c3159645b..5deeaf924ae 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp
@@ -7,14 +7,14 @@ namespace proton {
 
 namespace {
 
-const vespalib::string _G_documentMetaStoreName("[documentmetastore]");
+const vespalib::string documentMetaStoreName("[documentmetastore]");
 
 }
 
 const vespalib::string &
 DocumentMetaStoreAttribute::getFixedName()
 {
-    return _G_documentMetaStoreName;
+    return documentMetaStoreName;
 }
 
 DocumentMetaStoreAttribute::DocumentMetaStoreAttribute(const vespalib::string &name)
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
index 1a5f9c0077e..36408a01c17 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
@@ -14,12 +14,11 @@ namespace proton {
 class DocumentMetaStoreAttribute : public search::NotImplementedAttribute
 {
 public:
-    DocumentMetaStoreAttribute(const vespalib::string &name=getFixedName());
+    explicit DocumentMetaStoreAttribute(const vespalib::string &name);
     ~DocumentMetaStoreAttribute() override;
 
     static const vespalib::string &getFixedName();
 
-    // Implements IAttributeVector
     size_t getFixedWidth() const override {
         return document::GlobalId::LENGTH;
     }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
index f332ca5ec26..ce1f30593c5 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
@@ -28,7 +28,7 @@ ResultProcessor::Result::~Result() = default;
 ResultProcessor::Sort::Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss)
     : sorter(FastS_DefaultResultSorter::instance()),
       _ucaFactory(std::make_unique<search::uca::UcaConverterFactory>()),
-      sortSpec(partitionId, doom, *_ucaFactory)
+      sortSpec("[no-metastore]", partitionId, doom, *_ucaFactory)
 {
     if (!ss.empty() && sortSpec.Init(ss.c_str(), ac)) {
         sorter = &sortSpec;
@@ -46,9 +46,9 @@ ResultProcessor::Context::~Context() = default;
 
 void
 ResultProcessor::GroupingSource::merge(Source &s) {
-    GroupingSource &rhs = static_cast<GroupingSource&>(s);
-    assert((ctx == 0) == (rhs.ctx == 0));
-    if (ctx != 0) {
+    auto &rhs = dynamic_cast<GroupingSource&>(s);
+    assert((ctx == nullptr) == (rhs.ctx == nullptr));
+    if (ctx != nullptr) {
         search::grouping::GroupingManager man(*ctx);
         man.merge(*rhs.ctx);
     }
@@ -112,7 +112,7 @@ ResultProcessor::extract_docid_ordering(const PartialResult &result) const
     }
     std::sort(list.begin(), list.end(), [](const auto &a, const auto &b){ return (a.first < b.first); });
     return list;
-};
+}
 
 ResultProcessor::Result::UP
 ResultProcessor::makeReply(PartialResultUP full_result)