Merge pull request #1549 from yahoo/geirst/wiring-of-lid-space-compaction-to-document-store

Geirst/wiring of lid space compaction to document store

9 files changed, 99 insertions, 10 deletions

diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp
index ef7f544e93e..45637828ab1 100644
--- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp
+++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp
@@ -36,6 +36,9 @@ struct NullDataStore : IDataStore {
         std::vector<DataStoreFileChunkStats> result;
         return result;
     }
+    virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; }
+    virtual bool canShrinkLidSpace() const override { return false; }
+    virtual void shrinkLidSpace() override {}
 };
 
 TEST_FFF("require that uncache docstore lookups are counted",
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h
index c3d0e36184e..5011670bd41 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h
@@ -13,8 +13,9 @@
 #include <vespa/searchcommon/common/undefinedvalues.h>
 #include <vespa/searchlib/common/address_space.h>
 #include <vespa/searchlib/common/bitvector.h>
-#include <vespa/searchlib/common/range.h>
+#include <vespa/searchlib/common/i_compactable_lid_space.h>
 #include <vespa/searchlib/common/identifiable.h>
+#include <vespa/searchlib/common/range.h>
 #include <vespa/searchlib/common/rcuvector.h>
 #include <vespa/searchlib/queryeval/searchiterator.h>
 #include <vespa/vespalib/objects/identifiable.h>
@@ -106,7 +107,8 @@ public:
 };
 
 class AttributeVector : public vespalib::Identifiable,
-                        public attribute::IAttributeVector
+                        public attribute::IAttributeVector,
+                        public common::ICompactableLidSpace
 {
 protected:
     using Config = search::attribute::Config;
@@ -691,11 +693,11 @@ public:
     bool hasPostings();
     virtual uint64_t getUniqueValueCount() const;
     virtual uint64_t getTotalValueCount() const;
-    virtual void compactLidSpace(uint32_t wantedLidLimit);
+    virtual void compactLidSpace(uint32_t wantedLidLimit) override;
     virtual void clearDocs(DocId lidLow, DocId lidLimit);
     bool wantShrinkLidSpace(void) const { return _committedDocIdLimit < getNumDocs(); }
-    virtual bool canShrinkLidSpace() const;
-    void shrinkLidSpace();
+    virtual bool canShrinkLidSpace() const override;
+    virtual void shrinkLidSpace() override;
     virtual void onShrinkLidSpace();
 
     void setInterlock(const std::shared_ptr<attribute::Interlock> &interlock);
diff --git a/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h
new file mode 100644
index 00000000000..65b04570dcb
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h
@@ -0,0 +1,33 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search {
+namespace common {
+
+/**
+ * Interface for a component that has a lid space that can be compacted and shrinked.
+ */     
+struct ICompactableLidSpace {
+    virtual ~ICompactableLidSpace() {}
+
+    /**
+     * Compacts the lid space down to the wanted given doc id limit.
+     * After this, the remaining lid space is a candidate for shrinking (freeing of memory resources).
+     */  
+    virtual void compactLidSpace(uint32_t wantedDocLidLimit) = 0;
+
+    /**
+     * Returns whether this lid space can be shrinked down to the wanted doc id limit.
+     */ 
+    virtual bool canShrinkLidSpace() const = 0;
+
+    /**
+     * Shrinks this lid space down to the wanted doc id limit (frees memory resources).
+     */
+    virtual void shrinkLidSpace() = 0;
+};
+
+}
+}
+
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index 230da42244c..bf360784a56 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -508,5 +508,23 @@ CacheStats DocumentStore::getCacheStats() const {
     return singleStats;
 }
 
+void
+DocumentStore::compactLidSpace(uint32_t wantedDocLidLimit)
+{
+    _backingStore.compactLidSpace(wantedDocLidLimit);
+}
+
+bool
+DocumentStore::canShrinkLidSpace() const
+{
+    return _backingStore.canShrinkLidSpace();
+}
+
+void
+DocumentStore::shrinkLidSpace()
+{
+    _backingStore.shrinkLidSpace();
+}
+
 } // namespace search
 
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h
index 5d1b1f8137d..893e7e7c3d3 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.h
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h
@@ -164,6 +164,13 @@ public:
     virtual std::vector<DataStoreFileChunkStats>
     getFileChunkStats() const override;
 
+    /**
+     * Implements common::ICompactableLidSpace
+     */
+    virtual void compactLidSpace(uint32_t wantedDocLidLimit) override;
+    virtual bool canShrinkLidSpace() const override;
+    virtual void shrinkLidSpace() override;
+
 private:
     bool useCache() const;
 
diff --git a/searchlib/src/vespa/searchlib/docstore/idatastore.h b/searchlib/src/vespa/searchlib/docstore/idatastore.h
index e09b50db08f..9351a0f7802 100644
--- a/searchlib/src/vespa/searchlib/docstore/idatastore.h
+++ b/searchlib/src/vespa/searchlib/docstore/idatastore.h
@@ -3,9 +3,10 @@
 #pragma once
 
 #include "data_store_file_chunk_stats.h"
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/searchlib/util/memoryusage.h>
 #include <vespa/fastos/timestamp.h>
+#include <vespa/searchlib/common/i_compactable_lid_space.h>
+#include <vespa/searchlib/util/memoryusage.h>
+#include <vespa/vespalib/stllike/string.h>
 #include <vector>
 
 namespace vespalib { class DataBuffer; }
@@ -34,7 +35,7 @@ public:
  * Changes are held in memory until flush() is called.
  * A sync token is associated with each flush().
  **/
-class IDataStore
+class IDataStore : public common::ICompactableLidSpace
 {
 public:
     typedef std::vector<uint32_t> LidVector;
diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h
index 2ce8c64a694..0d0e18a0d82 100644
--- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h
+++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h
@@ -3,8 +3,9 @@
 #pragma once
 
 #include <vespa/document/fieldvalue/document.h>
-#include <vespa/searchlib/query/base.h>
+#include <vespa/searchlib/common/i_compactable_lid_space.h>
 #include <vespa/searchlib/docstore/idatastore.h>
+#include <vespa/searchlib/query/base.h>
 
 namespace search {
 
@@ -47,7 +48,7 @@ private:
  * updates will be held in memory until flush() is called.
  * Uses a Local ID as key.
  **/
-class IDocumentStore
+class IDocumentStore : public common::ICompactableLidSpace
 {
 public:
     /**
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
index a3eb5ee03f7..703982472e9 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
@@ -1102,4 +1102,21 @@ LogDataStore::getFileChunkStats() const
     return std::move(result);
 }
 
+void
+LogDataStore::compactLidSpace(uint32_t wantedDocLidLimit)
+{
+    (void) wantedDocLidLimit;
+}
+
+bool
+LogDataStore::canShrinkLidSpace() const
+{
+    return false;
+}
+
+void
+LogDataStore::shrinkLidSpace()
+{
+}
+
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
index 0fb5a582f5a..614d1103830 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h
@@ -195,6 +195,13 @@ public:
     virtual std::vector<DataStoreFileChunkStats>
     getFileChunkStats() const override;
 
+    /**
+     * Implements common::ICompactableLidSpace
+     */
+    virtual void compactLidSpace(uint32_t wantedDocLidLimit) override;
+    virtual bool canShrinkLidSpace() const override;
+    virtual void shrinkLidSpace() override;
+
 private:
     class WrapVisitor;
     class WrapVisitorProgress;