Compact lid space on source selector.

12 files changed, 80 insertions, 4 deletions

diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index c11bea14863..0f9df1fc594 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -140,12 +140,14 @@ struct MyIndexWriter : public test::MockIndexWriter
     MyLidVector _removes;
     int _heartBeatCount;
     uint32_t _commitCount;
+    uint32_t _wantedLidLimit;
     MyTracer &_tracer;
     MyIndexWriter(MyTracer &tracer)
         : test::MockIndexWriter(IIndexManager::SP(new test::MockIndexManager())),
           _removes(),
           _heartBeatCount(0),
           _commitCount(0),
+          _wantedLidLimit(0),
           _tracer(tracer)
     {}
     virtual void put(SerialNum serialNum, const document::Document &doc,
@@ -164,6 +166,9 @@ struct MyIndexWriter : public test::MockIndexWriter
         _tracer.traceCommit(indexAdapterTypeName, serialNum);
     }
     virtual void heartBeat(SerialNum) override { ++_heartBeatCount; }
+    void compactLidSpace(SerialNum, uint32_t lidLimit) override {
+        _wantedLidLimit = lidLimit;
+    }
 };
 
 struct MyGidToLidChangeHandler : public MockGidToLidChangeHandler
@@ -1144,7 +1149,7 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc
     f.compactLidSpaceAndWait(2);
     // performIndexForceCommit in index thread, then completion callback
     // in master thread.
-    EXPECT_TRUE(assertThreadObserver(7, 4, 3, f.writeServiceObserver()));
+    EXPECT_TRUE(assertThreadObserver(7, 5, 3, f.writeServiceObserver()));
     EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
@@ -1162,7 +1167,7 @@ TEST_F("require that compactLidSpace() doesn't propagate to "
     op.setSerialNum(0);
     f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); });
     // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread
-    EXPECT_TRUE(assertThreadObserver(6, 3, 3, f.writeServiceObserver()));
+    EXPECT_TRUE(assertThreadObserver(6, 4, 3, f.writeServiceObserver()));
     EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
@@ -1176,6 +1181,13 @@ TEST_F("require that compactLidSpace() propagates to attributeadapter",
     EXPECT_EQUAL(2u, f.maw._wantedLidLimit);
 }
 
+TEST_F("require that compactLidSpace() propagates to index writer",
+       SearchableFeedViewFixture)
+{
+    f.populateBeforeCompactLidSpace();
+    f.compactLidSpaceAndWait(2);
+    EXPECT_EQUAL(2u, f.miw._wantedLidLimit);
+}
 
 TEST_F("require that commit is called if visibility delay is 0",
        SearchableFeedViewFixture)
diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp
index 0d9fe42b978..8f5d998b77a 100644
--- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp
+++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp
@@ -33,8 +33,11 @@ struct MyIndexManager : public test::MockIndexManager
     SerialNum current;
     SerialNum flushed;
     SerialNum commitSerial;
+    uint32_t  wantedLidLimit;
+    SerialNum compactSerial;
     MyIndexManager() : puts(), removes(), current(0), flushed(0),
-                       commitSerial(0)
+                       commitSerial(0),
+                       wantedLidLimit(0), compactSerial(0)
     {
     }
     std::string getPut(uint32_t lid) {
@@ -62,6 +65,10 @@ struct MyIndexManager : public test::MockIndexManager
     virtual SerialNum getFlushedSerialNum() const override {
         return flushed;
     }
+    void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override {
+        wantedLidLimit = lidLimit;
+        compactSerial = serialNum;
+    }
 };
 
 struct Fixture
@@ -95,7 +102,7 @@ struct Fixture
     }
 };
 
-TEST_F("require that index adapter ignores old operations", Fixture)
+TEST_F("require that index writer ignores old operations", Fixture)
 {
     f.mim.flushed = 10;
     f.put(8, 1);
@@ -110,6 +117,13 @@ TEST_F("require that commit is forwarded to index manager", Fixture)
     EXPECT_EQUAL(10u, f.mim.commitSerial);
 }
 
+TEST_F("require that compactLidSpace is forwarded to index manager", Fixture)
+{
+    f.iw.compactLidSpace(4, 2);
+    EXPECT_EQUAL(2u, f.mim.wantedLidLimit);
+    EXPECT_EQUAL(4u, f.mim.compactSerial);
+}
+
 TEST_MAIN()
 {
     TEST_RUN_ALL();
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
index c025bd257a2..d9f0a256faf 100644
--- a/searchcore/src/tests/proton/index/indexmanager_test.cpp
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -145,6 +145,14 @@ struct Fixture {
                           });
         _writeService.indexFieldWriter().sync();
     }
+    void removeDocument(uint32_t docId) {
+        SerialNum serialNum = ++_serial_num;
+        removeDocument(docId, serialNum);
+    }
+    void compactLidSpace(uint32_t lidLimit) {
+        SerialNum serialNum = ++_serial_num;
+        runAsIndex([&]() { _index_manager->compactLidSpace(lidLimit, serialNum); });
+    }
     void assertStats(uint32_t expNumDiskIndexes,
                      uint32_t expNumMemoryIndexes,
                      SerialNum expLastiskIndexSerialNum,
@@ -715,6 +723,18 @@ TEST_F("require that indexes manager stats can be generated", Fixture)
     TEST_DO(f.assertStats(1, 1, 1, 2));
 }
 
+TEST_F("require that compactLidSpace works", Fixture)
+{
+    Schema empty_schema;
+    f.addDocument(1);
+    f.addDocument(2);
+    f.removeDocument(2);
+    auto fsc = f._index_manager->getMaintainer().getSourceCollection();
+    EXPECT_EQUAL(3u, fsc->getSourceSelector().getDocIdLimit());
+    f.compactLidSpace(2);
+    EXPECT_EQUAL(2u, fsc->getSourceSelector().getDocIdLimit());
+}
+
 }  // namespace
 
 TEST_MAIN() {
diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h
index e42b8282941..9dc437e6460 100644
--- a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h
@@ -27,6 +27,7 @@ public:
     virtual void remove(search::SerialNum serialNum, const search::DocumentIdT lid) = 0;
     virtual void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) = 0;
     virtual void heartBeat(search::SerialNum serialNum) = 0;
+    virtual void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp
index 42480826d2b..ea7fcd99a3b 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp
@@ -67,4 +67,10 @@ IndexWriter::heartBeat(search::SerialNum serialNum)
     _mgr->heartBeat(serialNum);
 }
 
+void
+IndexWriter::compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid)
+{
+    _mgr->compactLidSpace(lid, serialNum);
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h
index ca8c120ecb5..c231a43e06e 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h
@@ -31,6 +31,7 @@ public:
 
     virtual void
     heartBeat(search::SerialNum serialNum) override;
+    void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
index 82e32b0c26b..2139e34818f 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
@@ -89,5 +89,11 @@ IndexManager::IndexManager(const vespalib::string &baseDir,
 
 IndexManager::~IndexManager() = default;
 
+void
+IndexManager::compactLidSpace(uint32_t lidLimit, SerialNum serialNum)
+{
+    _maintainer.compactLidSpace(lidLimit, serialNum);
+}
+
 } // namespace proton
 
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
index 15872583135..4f2d03c81e6 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
@@ -98,6 +98,7 @@ public:
     void heartBeat(SerialNum serialNum) override {
         _maintainer.heartBeat(serialNum);
     }
+    void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override;
 
     SerialNum getCurrentSerialNum() const override {
         return _maintainer.getCurrentSerialNum();
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
index 28b1c407429..eab118c1188 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
@@ -6,6 +6,7 @@
 #include "removedonecontext.h"
 #include <vespa/searchcore/proton/common/feedtoken.h>
 #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
+#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h>
 #include <vespa/searchlib/common/isequencedtaskexecutor.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <vespa/vespalib/util/closuretask.h>
@@ -204,6 +205,17 @@ SearchableFeedView::performIndexForceCommit(SerialNum serialNum, OnForceCommitDo
 }
 
 void
+SearchableFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
+{
+    Parent::handleCompactLidSpace(op);
+    _writeService.index().execute(
+            makeLambdaTask([this, &op]() {
+                               _indexWriter->compactLidSpace(op.getSerialNum(), op.getLidLimit());
+                           }));
+    _writeService.index().sync();
+}
+
+void
 SearchableFeedView::forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone)
 {
     Parent::forceCommit(serialNum, onCommitDone);
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
index ed3ba6740b1..84de5d8c853 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h
@@ -76,6 +76,7 @@ public:
 
     ~SearchableFeedView() override;
     const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; }
+    void handleCompactLidSpace(const CompactLidSpaceOperation &op) override;
     void sync() override;
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h
index 9e105ef8a7f..6395ec05671 100644
--- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h
@@ -28,6 +28,7 @@ struct MockIndexManager : public searchcorespi::IIndexManager
     }
     virtual void setSchema(const Schema &, SerialNum) override {}
     virtual void heartBeat(SerialNum) override {}
+    void compactLidSpace(uint32_t, SerialNum) override {}
     virtual void setMaxFlushed(uint32_t) override { }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h
index 6e2df177dd7..8fe180533dc 100644
--- a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h
@@ -20,6 +20,7 @@ struct MockIndexWriter : public IIndexWriter
     virtual void remove(search::SerialNum, const search::DocumentIdT) override {}
     virtual void commit(search::SerialNum, OnWriteDoneType) override {}
     virtual void heartBeat(search::SerialNum) override {}
+    void compactLidSpace(search::SerialNum, const search::DocumentIdT) override {}
 };
 
 } // namespace test