Revert "Revert "Toregge/add shrink lid space flush target""

7 files changed, 93 insertions, 29 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index 65fbd6b8e4f..ade50306dc4 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -14,6 +14,7 @@ LOG_SETUP("attribute_manager_test");
 #include <vespa/searchcore/proton/attribute/i_attribute_functor.h>
 #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h>
 #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h>
+#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/initializer/initializer_task.h>
 #include <vespa/searchcore/proton/initializer/task_runner.h>
@@ -800,6 +801,45 @@ TEST_F("require that attribute vector of wrong type is dropped", BaseFixture)
     TEST_DO(assertCreateSerialNum(am2.mgr, "a6", 20));
 }
 
+void assertShrinkTargetSerial(proton::AttributeManager &mgr, const vespalib::string &name, search::SerialNum expSerialNum)
+{
+    auto shrinker = mgr.getShrinker(name);
+    EXPECT_EQUAL(expSerialNum, shrinker->getFlushedSerialNum());
+}
+
+TEST_F("require that we can guess flushed serial number for shrink flushtarget", BaseFixture)
+{
+    auto am1(std::make_shared<proton::AttributeManager>
+             (test_dir, "test.subdb", TuneFileAttributes(),
+              f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo));
+    am1->addAttribute({"a1", INT32_SINGLE}, 1);
+    am1->addAttribute({"a2", INT32_SINGLE}, 2);
+    TEST_DO(assertShrinkTargetSerial(*am1, "a1", 0));
+    TEST_DO(assertShrinkTargetSerial(*am1, "a2", 1));
+    am1->flushAll(10);
+    am1 = std::make_shared<proton::AttributeManager>
+          (test_dir, "test.subdb", TuneFileAttributes(),
+           f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo);
+    am1->addAttribute({"a1", INT32_SINGLE}, 1);
+    am1->addAttribute({"a2", INT32_SINGLE}, 2);
+    TEST_DO(assertShrinkTargetSerial(*am1, "a1", 10));
+    TEST_DO(assertShrinkTargetSerial(*am1, "a2", 10));
+}
+
+TEST_F("require that shrink flushtarget is handed over to new attribute manager", BaseFixture)
+{
+    auto am1(std::make_shared<proton::AttributeManager>
+             (test_dir, "test.subdb", TuneFileAttributes(),
+              f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo));
+    am1->addAttribute({"a1", INT32_SINGLE}, 4);
+    AttrSpecList newSpec;
+    newSpec.push_back(AttributeSpec("a1", INT32_SINGLE));
+    auto am2 = am1->create(AttrMgrSpec(newSpec, 5, 20));
+    auto am3 = std::dynamic_pointer_cast<AttributeManager>(am2);
+    TEST_DO(assertShrinkTargetSerial(*am3, "a1", 3));
+    EXPECT_EQUAL(am1->getShrinker("a1"), am3->getShrinker("a1"));
+}
+
 TEST_MAIN()
 {
     vespalib::rmdir(test_dir, true);
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
index 41ff853047d..4a3f0eb09b7 100644
--- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
@@ -100,5 +100,6 @@ TEST_F("require that reprocess with document populates attribute", Fixture)
 
 TEST_MAIN()
 {
+    vespalib::rmdir(TEST_DIR, true);
     TEST_RUN_ALL();
 }
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index 173d8dd2a2a..b0752e82c56 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -13,6 +13,7 @@ LOG_SETUP("attribute_test");
 #include <vespa/searchcore/proton/attribute/attributemanager.h>
 #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
 #include <vespa/searchcore/proton/test/attribute_utils.h>
+#include <vespa/searchcorespi/flush/iflushtarget.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/common/idestructorcallback.h>
@@ -550,6 +551,10 @@ TEST_F("require that filter attribute manager can filter attributes", FilterFixt
     f._filterMgr.getAttributeList(attrs);
     EXPECT_EQUAL(1u, attrs.size());
     EXPECT_EQUAL("a2", attrs[0]->getName());
+    searchcorespi::IFlushTarget::List targets = f._filterMgr.getFlushTargets();
+    EXPECT_EQUAL(2u, targets.size());
+    EXPECT_EQUAL("attribute.flush.a2", targets[0]->getName());
+    EXPECT_EQUAL("attribute.shrink.a2", targets[1]->getName());
 }
 
 TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture)
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
index 5375f7817ec..714e9076d16 100644
--- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
@@ -6,6 +6,7 @@
 #include <vespa/searchcore/proton/attribute/attributedisklayout.h>
 #include <vespa/searchcore/proton/attribute/attribute_writer.h>
 #include <vespa/searchcore/proton/attribute/flushableattribute.h>
+#include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/common/indexmetainfo.h>
@@ -389,9 +390,11 @@ Test::requireThatFlushTargetsCanBeRetrieved(void)
     f.addAttribute("a4");
     f.addAttribute("a5");
     std::vector<IFlushTarget::SP> ftl = am.getFlushTargets();
-    EXPECT_EQUAL(2u, ftl.size());
+    EXPECT_EQUAL(4u, ftl.size());
     EXPECT_EQUAL(am.getFlushable("a4").get(), ftl[0].get());
-    EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[1].get());
+    EXPECT_EQUAL(am.getShrinker("a4").get(),  ftl[1].get());
+    EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[2].get());
+    EXPECT_EQUAL(am.getShrinker("a5").get(),  ftl[3].get());
 }
 
 
@@ -528,7 +531,7 @@ Test::requireThatShrinkWorks()
     
     av->addDocs(1000 - av->getNumDocs());
     av->commit(10, 10);
-    IFlushTarget::SP ft = am.getFlushable("a10");
+    IFlushTarget::SP ft = am.getShrinker("a10");
     EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(),
                  ft->getApproxMemoryGain().getAfter());
     AttributeGuard::UP g = am.getAttribute("a10");
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index abfd80c9582..88409052189 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -685,23 +685,25 @@ assertTarget(const vespalib::string &name,
 TEST_F("require that flush targets can be retrieved", FastAccessFixture)
 {
 	IFlushTarget::List targets = getFlushTargets(f);
-	EXPECT_EQUAL(4u, targets.size());
-	EXPECT_EQUAL("subdb.attribute.attr1", targets[0]->getName());
-	EXPECT_EQUAL("subdb.documentmetastore", targets[1]->getName());
-	EXPECT_EQUAL("subdb.summary.compact", targets[2]->getName());
-	EXPECT_EQUAL("subdb.summary.flush", targets[3]->getName());
+	EXPECT_EQUAL(5u, targets.size());
+	EXPECT_EQUAL("subdb.attribute.flush.attr1", targets[0]->getName());
+	EXPECT_EQUAL("subdb.attribute.shrink.attr1", targets[1]->getName());
+	EXPECT_EQUAL("subdb.documentmetastore", targets[2]->getName());
+	EXPECT_EQUAL("subdb.summary.compact", targets[3]->getName());
+	EXPECT_EQUAL("subdb.summary.flush", targets[4]->getName());
 }
 
 TEST_F("require that flush targets can be retrieved", SearchableFixture)
 {
 	IFlushTarget::List targets = getFlushTargets(f);
-	EXPECT_EQUAL(6u, targets.size());
-	EXPECT_TRUE(assertTarget("subdb.attribute.attr1", FType::SYNC, FComponent::ATTRIBUTE, *targets[0]));
-	EXPECT_TRUE(assertTarget("subdb.documentmetastore", FType::SYNC, FComponent::ATTRIBUTE, *targets[1]));
-	EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[2]));
-	EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[3]));
-	EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[4]));
-	EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[5]));
+	EXPECT_EQUAL(7u, targets.size());
+	EXPECT_TRUE(assertTarget("subdb.attribute.flush.attr1", FType::SYNC, FComponent::ATTRIBUTE, *targets[0]));
+	EXPECT_TRUE(assertTarget("subdb.attribute.shrink.attr1", FType::GC, FComponent::ATTRIBUTE, *targets[1]));
+	EXPECT_TRUE(assertTarget("subdb.documentmetastore", FType::SYNC, FComponent::ATTRIBUTE, *targets[2]));
+	EXPECT_TRUE(assertTarget("subdb.memoryindex.flush", FType::FLUSH, FComponent::INDEX, *targets[3]));
+	EXPECT_TRUE(assertTarget("subdb.memoryindex.fusion", FType::GC, FComponent::INDEX, *targets[4]));
+	EXPECT_TRUE(assertTarget("subdb.summary.compact", FType::GC, FComponent::DOCUMENT_STORE, *targets[5]));
+	EXPECT_TRUE(assertTarget("subdb.summary.flush", FType::SYNC, FComponent::DOCUMENT_STORE, *targets[6]));
 }
 
 TEST_F("require that only fast-access attributes are instantiated", FastAccessOnlyFixture)
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index bdc0fc5f15c..280b617246b 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -690,6 +690,7 @@ struct FixtureBase
     void assertChangeNotified(document::GlobalId gid, uint32_t expLid) {
         _gidToLidChangeHandler->assertLid(gid, expLid);
     }
+    void populateBeforeCompactLidSpace();
 };
 
 
@@ -720,6 +721,14 @@ FixtureBase::FixtureBase(TimeStamp visibilityDelay)
 FixtureBase::~FixtureBase() {
     _writeServiceReal.sync();
 }
+
+void
+FixtureBase::populateBeforeCompactLidSpace()
+{
+    putAndWait(makeDummyDocs(0, 2, 1000));
+    removeAndWait(makeDummyDocs(1, 1, 2000));
+}
+
 struct SearchableFeedViewFixture : public FixtureBase
 {
     SearchableFeedView fv;
@@ -1135,17 +1144,16 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc
        "blocks lid space shrinkage until generation is no longer used",
        SearchableFeedViewFixture)
 {
-    EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver()));
-    CompactLidSpaceOperation op(0, 99);
-    op.setSerialNum(1);
-    f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); });
+    f.populateBeforeCompactLidSpace();
+    EXPECT_TRUE(assertThreadObserver(5, 3, f.writeServiceObserver()));
+    f.compactLidSpaceAndWait(2);
     // performIndexForceCommit in index thread, then completion callback
     // in master thread.
-    EXPECT_TRUE(assertThreadObserver(3, 1, f.writeServiceObserver()));
-    EXPECT_EQUAL(99u, f.metaStoreObserver()._compactLidSpaceLidLimit);
-    EXPECT_EQUAL(99u, f.getDocumentStore()._compactLidSpaceLidLimit);
+    EXPECT_TRUE(assertThreadObserver(7, 4, f.writeServiceObserver()));
+    EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit);
+    EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
-    EXPECT_EQUAL(99u, f._docIdLimit.get());
+    EXPECT_EQUAL(2u, f._docIdLimit.get());
 }
 
 TEST_F("require that compactLidSpace() doesn't propagate to "
@@ -1153,12 +1161,13 @@ TEST_F("require that compactLidSpace() doesn't propagate to "
        "blocks lid space shrinkage until generation is no longer used",
        SearchableFeedViewFixture)
 {
-    EXPECT_TRUE(assertThreadObserver(1, 0, f.writeServiceObserver()));
-    CompactLidSpaceOperation op(0, 99);
+    f.populateBeforeCompactLidSpace();
+    EXPECT_TRUE(assertThreadObserver(5, 3, f.writeServiceObserver()));
+    CompactLidSpaceOperation op(0, 2);
     op.setSerialNum(0);
     f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); });
     // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread
-    EXPECT_TRUE(assertThreadObserver(2, 0, f.writeServiceObserver()));
+    EXPECT_TRUE(assertThreadObserver(6, 3, f.writeServiceObserver()));
     EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
@@ -1167,8 +1176,9 @@ TEST_F("require that compactLidSpace() doesn't propagate to "
 TEST_F("require that compactLidSpace() propagates to attributeadapter",
        FastAccessFeedViewFixture)
 {
-    f.runInMaster([&] () { f.fv.handleCompactLidSpace(CompactLidSpaceOperation(0, 99)); });
-    EXPECT_EQUAL(99u, f.maw._wantedLidLimit);
+    f.populateBeforeCompactLidSpace();
+    f.compactLidSpaceAndWait(2);
+    EXPECT_EQUAL(2u, f.maw._wantedLidLimit);
 }
 
 
diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
index 3269fe84dcd..12a3ae7fe5e 100644
--- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
+++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp
@@ -22,6 +22,7 @@ struct MFT : public test::DummyFlushTarget
 };
 
 struct AttributeFlush : public MFT { AttributeFlush() : MFT(FTT::SYNC, FTC::ATTRIBUTE) {} };
+struct AttributeShrink : public MFT { AttributeShrink() : MFT(FTT::GC, FTC::ATTRIBUTE) {} };
 struct MemoryIndexFlush : public MFT { MemoryIndexFlush() : MFT(FTT::FLUSH, FTC::INDEX) {} };
 struct DiskIndexFusion : public MFT { DiskIndexFusion() : MFT(FTT::GC, FTC::INDEX) {} };
 struct DocStoreFlush : public MFT { DocStoreFlush() : MFT(FTT::SYNC, FTC::DOCUMENT_STORE) {} };
@@ -93,14 +94,16 @@ TEST_F("require that known flush targets are tracked", Fixture)
     input.push_back(IFlushTarget::SP(new DiskIndexFusion()));
     input.push_back(IFlushTarget::SP(new DocStoreFlush()));
     input.push_back(IFlushTarget::SP(new DocStoreCompaction()));
+    input.push_back(IFlushTarget::SP(new AttributeShrink()));
 
     IFlushTarget::List output = f._trackers.trackFlushTargets(input);
-    EXPECT_EQUAL(5u, output.size());
+    EXPECT_EQUAL(6u, output.size());
     EXPECT_TRUE(assertFlushTarget(f._trackers.getAttributeFlush(), *output[0]));
     EXPECT_TRUE(assertFlushTarget(f._trackers.getMemoryIndexFlush(), *output[1]));
     EXPECT_TRUE(assertFlushTarget(f._trackers.getDiskIndexFusion(), *output[2]));
     EXPECT_TRUE(assertFlushTarget(f._trackers.getDocumentStoreFlush(), *output[3]));
     EXPECT_TRUE(assertFlushTarget(f._trackers.getDocumentStoreCompact(), *output[4]));
+    EXPECT_TRUE(assertFlushTarget(f._trackers.getAttributeFlush(), *output[5]));
 }
 
 TEST_F("require that un-known flush targets are not tracked", Fixture)