Wire lid space compaction down to document store.

8 files changed, 84 insertions, 53 deletions

diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index e948bb518d3..ba5d22dca28 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -20,6 +20,7 @@
 #include <vespa/searchcore/proton/server/summaryadapter.h>
 #include <vespa/searchcore/proton/server/reconfig_params.h>
 #include <vespa/searchcore/proton/test/documentdb_config_builder.h>
+#include <vespa/searchcore/proton/test/mock_summary_adapter.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/transactionlog/nosyncproxy.h>
 #include <vespa/vespalib/io/fileutil.h>
@@ -228,20 +229,7 @@ Fixture::initViewSet(ViewSet &views)
 }
 
 
-struct MySummaryAdapter : public ISummaryAdapter
-{
-    virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) {}
-    virtual void remove(search::SerialNum, const search::DocumentIdT) {}
-    virtual void heartBeat(search::SerialNum) {}
-    virtual const search::IDocumentStore &getDocumentStore() const {
-        const search::IDocumentStore *store = NULL;
-        return *store;
-    }
-    virtual std::unique_ptr<document::Document> get(const search::DocumentIdT,
-            const document::DocumentTypeRepo &) {
-        return std::unique_ptr<document::Document>();
-    }
-};
+using MySummaryAdapter = test::MockSummaryAdapter;
 
 struct MyFastAccessFeedView
 {
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index 5d932cb7dfc..60fc54cd388 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -22,6 +22,7 @@ LOG_SETUP("feedview_test");
 #include <vespa/searchcore/proton/test/dummy_summary_manager.h>
 #include <vespa/searchcore/proton/test/mock_index_writer.h>
 #include <vespa/searchcore/proton/test/mock_index_manager.h>
+#include <vespa/searchcore/proton/test/mock_summary_adapter.h>
 #include <vespa/searchcore/proton/test/thread_utils.h>
 #include <vespa/searchcore/proton/test/threading_service_observer.h>
 #include <vespa/searchlib/docstore/cachestats.h>
@@ -181,10 +182,12 @@ struct MyDocumentStore : public test::DummyDocumentStore
     typedef std::map<DocumentIdT, document::Document::SP> DocMap;
     DocMap           _docs;
     uint64_t         _lastSyncToken;
+    uint32_t         _compactLidSpaceLidLimit;
     MyDocumentStore()
         : test::DummyDocumentStore("."),
           _docs(),
-          _lastSyncToken(0)
+          _lastSyncToken(0),
+          _compactLidSpaceLidLimit(0)
     {}
     virtual Document::UP read(DocumentIdT lid, const document::DocumentTypeRepo &) const {
         DocMap::const_iterator itr = _docs.find(lid);
@@ -206,6 +209,9 @@ struct MyDocumentStore : public test::DummyDocumentStore
         return syncToken;
     }
     virtual uint64_t lastSyncToken() const { return _lastSyncToken; }
+    virtual void compactLidSpace(uint32_t wantedDocLidLimit) override {
+        _compactLidSpaceLidLimit = wantedDocLidLimit;
+    }
 };
 
 struct MySummaryManager : public test::DummySummaryManager
@@ -215,11 +221,11 @@ struct MySummaryManager : public test::DummySummaryManager
     virtual search::IDocumentStore &getBackingStore() { return _store; }
 };
 
-struct MySummaryAdapter : public ISummaryAdapter
+struct MySummaryAdapter : public test::MockSummaryAdapter
 {
     ISummaryManager::SP _sumMgr;
     MyDocumentStore    &_store;
-    MyLidVector           _removes;
+    MyLidVector         _removes;
 
     MySummaryAdapter()
         : _sumMgr(new MySummaryManager()),
@@ -227,26 +233,27 @@ struct MySummaryAdapter : public ISummaryAdapter
           _removes()
     {
     }
-    virtual void put(SerialNum serialNum, const document::Document &doc,
-                     const DocumentIdT lid) {
+    virtual void put(SerialNum serialNum, const document::Document &doc, const DocumentIdT lid) override {
         (void) serialNum;
         _store.write(serialNum, doc, lid);
     }
-    virtual void remove(SerialNum serialNum, const DocumentIdT lid) {
+    virtual void remove(SerialNum serialNum, const DocumentIdT lid) override {
         LOG(info,
             "MySummaryAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)",
             serialNum, lid);
         _store.remove(serialNum, lid);
         _removes.push_back(lid);
     }
-    virtual void heartBeat(SerialNum) {}
-    virtual const search::IDocumentStore &getDocumentStore() const {
+    virtual const search::IDocumentStore &getDocumentStore() const override {
         return _store;
     }
     virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid,
-                                                    const document::DocumentTypeRepo &repo) {
+                                                    const document::DocumentTypeRepo &repo) override {
         return _store.read(lid, repo);
     }
+    virtual void compactLidSpace(uint32_t wantedDocIdLimit) override {
+        _store.compactLidSpace(wantedDocIdLimit);
+    }
 };
 
 struct MyAttributeWriter : public IAttributeWriter
@@ -538,6 +545,9 @@ struct FixtureBase
     const IDocumentMetaStore &getMetaStore() const {
         return _dmsc->get();
     }
+    const MyDocumentStore &getDocumentStore() const {
+        return msa._store;
+    }
 
     BucketDBOwner::Guard getBucketDB() const {
         return getMetaStore().getBucketDB().takeGuard();
@@ -1028,7 +1038,7 @@ TEST_F("require that update() to tensor attribute updates attribute and document
     requireThatUpdateUpdatesAttributeAndDocumentStore(f, "a3");
 }
 
-TEST_F("require that compactLidSpace() propagates to document meta store and "
+TEST_F("require that compactLidSpace() propagates to document meta store and document store and "
        "blocks lid space shrinkage until generation is no longer used",
        SearchableFeedViewFixture)
 {
@@ -1040,12 +1050,13 @@ TEST_F("require that compactLidSpace() propagates to document meta store and "
     // in master thread.
     EXPECT_TRUE(assertThreadObserver(3, 1, f.writeServiceObserver()));
     EXPECT_EQUAL(99u, f.metaStoreObserver()._compactLidSpaceLidLimit);
+    EXPECT_EQUAL(99u, f.getDocumentStore()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
     EXPECT_EQUAL(99u, f._docIdLimit.get());
 }
 
 TEST_F("require that compactLidSpace() doesn't propagate to "
-       "document meta store and "
+       "document meta store and document store and "
        "blocks lid space shrinkage until generation is no longer used",
        SearchableFeedViewFixture)
 {
@@ -1056,6 +1067,7 @@ TEST_F("require that compactLidSpace() doesn't propagate to "
     // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread
     EXPECT_TRUE(assertThreadObserver(2, 0, f.writeServiceObserver()));
     EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit);
+    EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit);
     EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt);
 }
 
diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
index 672bcd02529..eadc7566162 100644
--- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp
@@ -11,6 +11,7 @@
 #include <vespa/searchcore/proton/server/executorthreadingservice.h>
 #include <vespa/searchcore/proton/server/storeonlyfeedview.h>
 #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
+#include <vespa/searchcore/proton/test/mock_summary_adapter.h>
 #include <vespa/searchcore/proton/test/thread_utils.h>
 #include <vespa/searchcore/proton/common/commit_time_tracker.h>
 #include <vespa/searchlib/index/docbuilder.h>
@@ -37,7 +38,7 @@ using namespace proton;
 
 namespace {
 
-class MySummaryAdapter : public ISummaryAdapter {
+class MySummaryAdapter : public test::MockSummaryAdapter {
     int &_rm_count;
     int &_put_count;
     int &_heartbeat_count;
@@ -48,15 +49,9 @@ public:
           _put_count(put_count),
           _heartbeat_count(heartbeat_count) {
     }
-    virtual void put(SerialNum, const Document &, DocumentIdT)
-    { ++ _put_count; }
-    virtual void remove(SerialNum, DocumentIdT) { ++_rm_count; }
-    virtual void heartBeat(SerialNum) { ++_heartbeat_count; }
-    virtual const search::IDocumentStore &getDocumentStore() const
-    { return *reinterpret_cast<const search::IDocumentStore *>(0); }
-
-    virtual Document::UP get(const DocumentIdT, const DocumentTypeRepo &)
-    { return Document::UP(); }
+    virtual void put(SerialNum, const Document &, DocumentIdT) override { ++ _put_count; }
+    virtual void remove(SerialNum, DocumentIdT) override { ++_rm_count; }
+    virtual void heartBeat(SerialNum) override { ++_heartbeat_count; }
 };
 
 DocumentTypeRepo::SP myGetDocumentTypeRepo() {
diff --git a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h
index 4090cb970f2..165af7c3eee 100644
--- a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h
@@ -35,6 +35,8 @@ public:
     virtual std::unique_ptr<document::Document>
     get(const search::DocumentIdT lid,
         const document::DocumentTypeRepo &repo) = 0;
+
+    virtual void compactLidSpace(uint32_t wantedDocIdLimit) = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 31bcaca5014..caedb17fee3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -854,6 +854,9 @@ StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op)
         commitContext->holdUnblockShrinkLidSpace();
         forceCommit(serialNum, commitContext);
     }
+    if (useDocumentStore(serialNum)) {
+        _summaryAdapter->compactLidSpace(op.getLidLimit());
+    }
 }
 
 const ISimpleDocumentMetaStore *
diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h
index 3756ab04179..2e874ad4d25 100644
--- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h
@@ -22,25 +22,24 @@ public:
      */
     virtual void put(search::SerialNum serialNum,
                      const document::Document &doc,
-                     const search::DocumentIdT lid);
+                     const search::DocumentIdT lid) override;
     virtual void remove(search::SerialNum serialNum,
-                        const search::DocumentIdT lid);
+                        const search::DocumentIdT lid) override;
 
-    virtual void
-    heartBeat(search::SerialNum serialNum);
+    virtual void heartBeat(search::SerialNum serialNum) override;
 
-    virtual const search::IDocumentStore &
-    getDocumentStore(void) const
-    {
+    virtual const search::IDocumentStore &getDocumentStore() const {
         return _imgr->getBackingStore();
     }
 
-    virtual std::unique_ptr<document::Document>
-    get(const search::DocumentIdT lid,
-        const document::DocumentTypeRepo &repo)
-    {
+    virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid,
+                                                    const document::DocumentTypeRepo &repo) override {
         return _imgr->getBackingStore().read(lid, repo);
     }
+
+    virtual void compactLidSpace(uint32_t wantedDocIdLimit) override {
+        _mgr->getBackingStore().compactLidSpace(wantedDocIdLimit);
+    }
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h
index 2b43546be86..94c644778e5 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h
@@ -46,20 +46,19 @@ struct DummyDocumentStore : public search::IDocumentStore
                         search::IDocumentStoreVisitorProgress &,
                         const document::DocumentTypeRepo &) {}
 
-    virtual double
-    getVisitCost() const
-    {
-        return 1.0;
-    }
+    virtual double getVisitCost() const { return 1.0; }
     virtual search::DataStoreStorageStats getStorageStats() const override {
         return search::DataStoreStorageStats(0, 0, 0.0, 0, 0);
     }
     virtual search::MemoryUsage getMemoryUsage() const override { return search::MemoryUsage(); }
-    virtual std::vector<search::DataStoreFileChunkStats>
-    getFileChunkStats() const override {
+    virtual std::vector<search::DataStoreFileChunkStats> getFileChunkStats() const override {
         std::vector<search::DataStoreFileChunkStats> result;
         return result;
     }
+
+    virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; }
+    virtual bool canShrinkLidSpace() const override { return false; }
+    virtual void shrinkLidSpace() override {}
 };
 
 } // namespace test
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_summary_adapter.h b/searchcore/src/vespa/searchcore/proton/test/mock_summary_adapter.h
new file mode 100644
index 00000000000..7fa750a9dde
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_summary_adapter.h
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/searchcore/proton/server/isummaryadapter.h>
+
+namespace proton {
+
+namespace test {
+
+/**
+ * Mock of the ISummaryAdapter interface used for unit testing.
+ */
+struct MockSummaryAdapter : public ISummaryAdapter
+{
+    virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {}
+    virtual void remove(search::SerialNum, const search::DocumentIdT) override {}
+    virtual void heartBeat(search::SerialNum) override {}
+    virtual const search::IDocumentStore &getDocumentStore() const override {
+        const search::IDocumentStore *store = NULL;
+        return *store;
+    }
+    virtual std::unique_ptr<document::Document> get(const search::DocumentIdT,
+                                                    const document::DocumentTypeRepo &) override {
+        return std::unique_ptr<document::Document>();
+    }
+    virtual void compactLidSpace(uint32_t wantedDocIdLimit) override {
+        (void) wantedDocIdLimit;
+    }
+};
+
+} // namespace test
+
+} // namespace proton