Merge pull request #17207 from vespa-engine/toregge/compact-document-meta-store-gid-to-lid-map-when-needed

Compact DocumentMetaStore _gidToLidMap when dead bytes ratio is

5 files changed, 67 insertions, 7 deletions

diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
index 4e04677f972..84dadff0104 100644
--- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
@@ -2091,6 +2091,35 @@ TEST(DocumentMetaStoreTest, call_to_remove_is_notified)
     EXPECT_EQ(1, listener->remove_cnt);
 }
 
+TEST(DocumentMetaStoreTest, gid_to_lid_map_can_be_compacted)
+{
+    auto dms = std::make_shared<DocumentMetaStore>(createBucketDB());
+    dms->constructFreeList();
+    static constexpr uint32_t full_size = 1000;
+    for (uint32_t i = 1; i < full_size; ++i) {
+        addLid(*dms, i);
+    }
+    dms->commit(true);
+    AttributeGuard guard(dms);
+    remove(full_size - 1, 100, *dms);
+    dms->commit(true);
+    auto status_before = dms->getStatus();
+    EXPECT_LT(0, status_before.getOnHold());
+    guard = AttributeGuard();
+    dms->removeAllOldGenerations();
+    dms->commit(true);
+    auto status_middle = dms->getStatus();
+    EXPECT_LT(status_before.getDead(), status_middle.getDead());
+    EXPECT_EQ(0, status_middle.getOnHold());
+    for (uint32_t i = 0; i < 15; ++i) {
+        dms->commit(true);
+    }
+    auto status_after = dms->getStatus();
+    EXPECT_GT(status_before.getUsed(), status_after.getUsed());
+    EXPECT_GT(status_middle.getDead(), status_after.getDead());
+    EXPECT_EQ(0, status_after.getOnHold());
+}
+
 }
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
index 8152761c64d..25c0e29fbda 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
@@ -199,6 +199,36 @@ DocumentMetaStore::insert(GidToLidMapKey key, const RawDocumentMetaData &metaDat
     updateCommittedDocIdLimit();
 }
 
+namespace {
+
+// minimum dead bytes in gid to lid map before consider compaction
+constexpr size_t DEAD_BYTES_SLACK = 0x10000u;
+
+}
+
+bool
+DocumentMetaStore::consider_compact_gid_to_lid_map()
+{
+    auto &compaction_strategy = getConfig().getCompactionStrategy();
+    size_t used_bytes = _cached_gid_to_lid_map_memory_usage.usedBytes();
+    size_t dead_bytes = _cached_gid_to_lid_map_memory_usage.deadBytes();
+    bool compact_memory = ((dead_bytes >= DEAD_BYTES_SLACK) &&
+                           (used_bytes * compaction_strategy.getMaxDeadBytesRatio() < dead_bytes));
+    return compact_memory;
+}
+
+void
+DocumentMetaStore::onCommit()
+{
+    if (consider_compact_gid_to_lid_map()) {
+        _gidToLidMap.compact_worst();
+        _gid_to_lid_map_write_itr_prepare_serial_num = 0u;
+        _gid_to_lid_map_write_itr.begin(_gidToLidMap.getRoot());
+        this->incGeneration();
+        this->updateStat(true);
+    }
+}
+
 void
 DocumentMetaStore::onUpdateStat()
 {
@@ -207,7 +237,8 @@ DocumentMetaStore::onUpdateStat()
     size_t bvSize = _lidAlloc.getUsedLidsSize();
     usage.incAllocatedBytes(bvSize);
     usage.incUsedBytes(bvSize);
-    usage.merge(_gidToLidMap.getMemoryUsage());
+    _cached_gid_to_lid_map_memory_usage = _gidToLidMap.getMemoryUsage();
+    usage.merge(_cached_gid_to_lid_map_memory_usage);
     // the free lists are not taken into account here
     updateStatistics(_metaDataStore.size(),
                      _metaDataStore.size(),
@@ -400,7 +431,8 @@ DocumentMetaStore::DocumentMetaStore(BucketDBOwnerSP bucketDB,
       _shrinkLidSpaceBlockers(0),
       _subDbType(subDbType),
       _trackDocumentSizes(true),
-      _op_listener()
+      _op_listener(),
+      _cached_gid_to_lid_map_memory_usage()
 {
     ensureSpace(0);         // lid 0 is reserved
     setCommittedDocIdLimit(1u);         // lid 0 is reserved
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
index e5a009a6bd2..9b8fa26f9c7 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h
@@ -75,6 +75,7 @@ private:
     const SubDbType     _subDbType;
     bool                _trackDocumentSizes;
     OperationListenerSP _op_listener;
+    vespalib::MemoryUsage _cached_gid_to_lid_map_memory_usage;
 
     DocId getFreeLid();
     DocId peekFreeLid();
@@ -83,6 +84,8 @@ private:
 
     const GlobalId & getRawGid(DocId lid) const { return getRawMetaData(lid).getGid(); }
 
+    bool consider_compact_gid_to_lid_map();
+    void onCommit() override;
     void onUpdateStat() override;
 
     // Implements AttributeVector
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
index f4e936e663a..01b259615d8 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h
@@ -23,8 +23,6 @@ public:
     size_t getFixedWidth() const override {
         return document::GlobalId::LENGTH;
     }
-
-    void onCommit() override {}
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 945702d07bc..552d2b84179 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -783,9 +783,7 @@ StoreOnlyFeedView::heartBeat(SerialNum serialNum)
 {
     assert(_writeService.master().isCurrentThread());
     _metaStore.removeAllOldGenerations();
-    if (serialNum > _metaStore.getLastSerialNum()) {
-        _metaStore.commit(CommitParam(serialNum));
-    }
+    _metaStore.commit(CommitParam(serialNum));
     heartBeatSummary(serialNum);
     heartBeatIndexedFields(serialNum);
     heartBeatAttributes(serialNum);