Merge pull request #2476 from yahoo/toregge/add-document-meta-store-shrink-flushtarget

Add document meta store shrink flushtarget.

5 files changed, 33 insertions, 27 deletions

diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 88409052189..950b3d588f6 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -685,25 +685,27 @@ assertTarget(const vespalib::string &name,
 TEST_F("require that flush targets can be retrieved", FastAccessFixture)
 {
 	IFlushTarget::List targets = getFlushTargets(f);
-	EXPECT_EQUAL(5u, targets.size());
+	EXPECT_EQUAL(6u, targets.size());
 	EXPECT_EQUAL("subdb.attribute.flush.attr1", targets[0]->getName());
 	EXPECT_EQUAL("subdb.attribute.shrink.attr1", targets[1]->getName());
-	EXPECT_EQUAL("subdb.documentmetastore", targets[2]->getName());
-	EXPECT_EQUAL("subdb.summary.compact", targets[3]->getName());
-	EXPECT_EQUAL("subdb.summary.flush", targets[4]->getName());
+	EXPECT_EQUAL("subdb.documentmetastore.flush", targets[2]->getName());
+	EXPECT_EQUAL("subdb.documentmetastore.shrink", targets[3]->getName());
+	EXPECT_EQUAL("subdb.summary.compact", targets[4]->getName());
+	EXPECT_EQUAL("subdb.summary.flush", targets[5]->getName());
 }
 
 TEST_F("require that flush targets can be retrieved", SearchableFixture)
 {
 	IFlushTarget::List targets = getFlushTargets(f);
-	EXPECT_EQUAL(7u, targets.size());
+	EXPECT_EQUAL(8u, targets.size());
 	EXPECT_TRUE(assertTarget("subdb.attribute.flush.attr1", FType::SYNC, FComponent::ATTRIBUTE, *targets[0]));
 	EXPECT_TRUE(assertTarget("subdb.attribute.shrink.attr1", FType::GC, FComponent::ATTRIBUTE, *targets[1]));
-	EXPECT_TRUE(assertTarget("subdb.documentmetastore", FType::SYNC, FComponent::ATTRIBUTE, *targets[2]));
-	EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[3]));
-	EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[4]));
-	EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[5]));
-	EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[6]));
+	EXPECT_TRUE(assertTarget("subdb.documentmetastore.flush", FType::SYNC, FComponent::ATTRIBUTE, *targets[2]));
+	EXPECT_TRUE(assertTarget("subdb.documentmetastore.shrink", FType::GC, FComponent::ATTRIBUTE, *targets[3]));
+	EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[4]));
+	EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[5]));
+	EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[6]));
+	EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[7]));
 }
 
 TEST_F("require that only fast-access attributes are instantiated", FastAccessOnlyFixture)
diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
index d247c313ad4..e8484f1bf4d 100644
--- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp
@@ -2,7 +2,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/document/base/documentid.h>
 #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
-#include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
+#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h>
 #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
 #include <vespa/searchlib/attribute/attributefilesavetarget.h>
 #include <vespa/searchlib/fef/matchdatalayout.h>
@@ -1874,13 +1874,10 @@ TEST("requireThatShrinkViaFlushTargetWorks")
     HwInfo hwInfo;
     vespalib::rmdir("dmsflush", true);
     vespalib::mkdir("dmsflush");
-    IFlushTarget::SP ft(new DocumentMetaStoreFlushTarget(dms,
-                                                         dummyTlsSyncer,
-                                                         "dmsflush",
-                                                         tuneFileAttributes,
-                                                         fileHeaderContext,
-                                                         hwInfo));
-    
+    using Type = IFlushTarget::Type;
+    using Component = IFlushTarget::Component;
+    IFlushTarget::SP ft(std::make_shared<ShrinkLidSpaceFlushTarget>
+                        ("documentmetastore.shrink", Type::GC, Component::ATTRIBUTE, 0, dms));
     populate(10, *dms);
 
     uint32_t shrinkTarget = 5;
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
index e2fa972f926..f3158cc4661 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
@@ -71,9 +71,6 @@ Flusher(DocumentMetaStoreFlushTarget &dmsft,
 {
     DocumentMetaStore &dms = *_dmsft._dms;
     // Called by document db executor
-    if (dms.canShrinkLidSpace()) {
-        dms.shrinkLidSpace();
-    }
     _flushDir = writer.getSnapshotDir(syncToken);
     vespalib::string newBaseFileName(_flushDir + "/" + dms.getName());
     dms.setBaseFileName(newBaseFileName);
@@ -176,7 +173,7 @@ DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms,
                              const TuneFileAttributes &tuneFileAttributes,
                              const FileHeaderContext &fileHeaderContext,
                              const HwInfo &hwInfo)
-    : IFlushTarget("documentmetastore", Type::SYNC, Component::ATTRIBUTE),
+    : IFlushTarget("documentmetastore.flush", Type::SYNC, Component::ATTRIBUTE),
       _dms(dms),
       _tlsSyncer(tlsSyncer),
       _baseDir(baseDir),
@@ -209,11 +206,7 @@ IFlushTarget::MemoryGain
 DocumentMetaStoreFlushTarget::getApproxMemoryGain() const
 {
     int64_t used(_dms->getStatus().getUsed());
-    int64_t canFree = _dms->getEstimatedShrinkLidSpaceGain();
-    if (canFree > used) {
-        canFree = used;
-    }
-    return MemoryGain(used, used - canFree);
+    return MemoryGain(used, used);
 }
 
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index c09ba707295..b8a8e03ee66 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -14,6 +14,7 @@
 #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
 #include <vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h>
 #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
+#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h>
 #include <vespa/searchcore/proton/index/index_writer.h>
 #include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h>
 #include <vespa/searchcore/proton/metrics/metricswireservice.h>
@@ -128,6 +129,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx)
       _getSerialNum(ctx._getSerialNum),
       _tlsSyncer(ctx._writeService.master(), ctx._getSerialNum, ctx._tlSyncer),
       _dmsFlushTarget(),
+      _dmsShrinkTarget(),
       _subDbId(cfg._subDbId),
       _subDbType(cfg._subDbType),
       _fileHeaderContext(*this, ctx._fileHeaderContext, _docTypeName, _baseDir),
@@ -191,6 +193,7 @@ StoreOnlyDocSubDB::getOldestFlushedSerial()
 {
     SerialNum lowest(_iSummaryMgr->getBackingStore().lastSyncToken());
     lowest = std::min(lowest, _dmsFlushTarget->getFlushedSerialNum());
+    lowest = std::min(lowest, _dmsShrinkTarget->getFlushedSerialNum());
     return lowest;
 }
 
@@ -200,6 +203,7 @@ StoreOnlyDocSubDB::getNewestFlushedSerial()
 {
     SerialNum highest(_iSummaryMgr->getBackingStore().lastSyncToken());
     highest = std::max(highest, _dmsFlushTarget->getFlushedSerialNum());
+    highest = std::max(highest, _dmsShrinkTarget->getFlushedSerialNum());
     return highest;
 }
 
@@ -282,6 +286,13 @@ StoreOnlyDocSubDB::setupDocumentMetaStore(DocumentMetaStoreInitializerResult::SP
                                   baseDir,
                                   dmsResult->tuneFile(),
                                   _fileHeaderContext, _hwInfo));
+    using Type = IFlushTarget::Type;
+    using Component = IFlushTarget::Component;
+    _dmsShrinkTarget = std::make_shared<ShrinkLidSpaceFlushTarget>
+                       ("documentmetastore.shrink",
+                        Type::GC, Component::ATTRIBUTE,
+                        _flushedDocumentMetaStoreSerialNum,
+                        dms);
 }
 
 DocumentSubDbInitializer::UP
@@ -345,6 +356,7 @@ StoreOnlyDocSubDB::getFlushTargetsInternal()
 {
     IFlushTarget::List ret(_rSummaryMgr->getFlushTargets());
     ret.push_back(_dmsFlushTarget);
+    ret.push_back(_dmsShrinkTarget);
     return ret;
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index a6fe412b543..0a388a8343f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -28,6 +28,7 @@ class LegacyDocumentDBMetrics;
 class FeedHandler;
 class DocumentMetaStoreInitializerResult;
 class IDocumentSubDBOwner;
+class ShrinkLidSpaceFlushTarget;
 namespace initializer { class InitializerTask; }
 
 namespace bucketdb { class IBucketDBHandlerInitializer; }
@@ -166,6 +167,7 @@ private:
     const IGetSerialNum             &_getSerialNum;
     TlsSyncer                        _tlsSyncer;
     DocumentMetaStoreFlushTarget::SP _dmsFlushTarget;
+    std::shared_ptr<ShrinkLidSpaceFlushTarget> _dmsShrinkTarget;
 
     IFlushTargetList getFlushTargets() override;
 protected: