Wire in config for compaction strategy.

14 files changed, 61 insertions, 13 deletions

diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index d25a234c6f8..d07c25c2a9e 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -234,6 +234,7 @@ struct ParallelAttributeManager
     DocumentMetaStore::SP documentMetaStore;
     search::GrowStrategy attributeGrow;
     size_t attributeGrowNumDocs;
+    search::CompactionStrategy attribute_compaction_strategy;
     bool fastAccessAttributesOnly;
     std::shared_ptr<AttributeManager::SP> mgr;
     vespalib::ThreadStackExecutor masterExecutor;
@@ -252,13 +253,14 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria
       documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)),
       attributeGrow(),
       attributeGrowNumDocs(1),
+      attribute_compaction_strategy(),
       fastAccessAttributesOnly(false),
       mgr(std::make_shared<AttributeManager::SP>()),
       masterExecutor(1, 128 * 1024),
       master(masterExecutor),
       initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask,
                                                                 documentMetaStore, baseAttrMgr, attrCfg,
-                                                                attributeGrow, attributeGrowNumDocs,
+                                                                attributeGrow, attributeGrowNumDocs, attribute_compaction_strategy,
                                                                 fastAccessAttributesOnly, master, mgr))
 {
     documentMetaStore->setCommittedDocIdLimit(docIdLimit);
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index b67eaec1a8d..a9829f3e5f7 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -535,7 +535,7 @@ public:
     AttributeCollectionSpecFactory _factory;
     AttributeCollectionSpecTest(bool fastAccessOnly)
         : _builder(),
-          _factory(search::GrowStrategy(), 100, fastAccessOnly)
+          _factory(search::GrowStrategy(), 100, search::CompactionStrategy(), fastAccessOnly)
     {
         addAttribute("a1", false);
         addAttribute("a2", true);
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 9cdd984b152..64178303b40 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -124,7 +124,7 @@ struct MyStoreOnlyConfig
               SUB_NAME,
               BASE_DIR,
               search::GrowStrategy(),
-              0, 0, SubDbType::READY)
+              0, search::CompactionStrategy(), 0, SubDbType::READY)
     {
     }
 };
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index 17d0ef8ad37..74ec5c2fbef 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -306,6 +306,12 @@ documentdb[].allocation.amortizecount int default=10000
 ## used in multi-value attribute vectors to store underlying values.
 documentdb[].allocation.multivaluegrowfactor double default=0.2
 
+## The ratio of used bytes that can be dead before attempting to perform compaction.
+documentdb[].allocation.max_dead_bytes_ratio double default=0.2 restart
+
+## The ratio of used address space that can be dead before attempting to perform compaction.
+documentdb[].allocation.max_dead_address_space_ratio double default=0.2 restart
+
 ## The interval of when periodic tasks should be run
 periodic.interval double default=3600.0
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
index 23701bfdd5d..96db6205bac 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
@@ -11,9 +11,11 @@ namespace proton {
 AttributeCollectionSpecFactory::AttributeCollectionSpecFactory(
         const search::GrowStrategy &growStrategy,
         size_t growNumDocs,
+        const search::CompactionStrategy& compaction_strategy,
         bool fastAccessOnly)
     : _growStrategy(growStrategy),
       _growNumDocs(growNumDocs),
+      _compaction_strategy(compaction_strategy),
       _fastAccessOnly(fastAccessOnly)
 {
 }
@@ -35,6 +37,7 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg,
         }
         grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew);
         cfg.setGrowStrategy(grow);
+        cfg.setCompactionStrategy(_compaction_strategy);
         attrs.push_back(AttributeSpec(attr.name, cfg));
     }
     return std::make_unique<AttributeCollectionSpec>(attrs, docIdLimit, serialNum);
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
index bfa6681d6f2..9e877bedf28 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
@@ -4,6 +4,7 @@
 
 #include "attribute_collection_spec.h"
 #include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchcommon/common/compaction_strategy.h>
 #include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/config-attributes.h>
@@ -21,11 +22,13 @@ private:
 
     const search::GrowStrategy _growStrategy;
     const size_t               _growNumDocs;
+    const search::CompactionStrategy _compaction_strategy;
     const bool                 _fastAccessOnly;
 
 public:
     AttributeCollectionSpecFactory(const search::GrowStrategy &growStrategy,
                                    size_t growNumDocs,
+                                   const search::CompactionStrategy& compaction_strategy,
                                    bool fastAccessOnly);
 
     AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg,
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
index 9198cfdafab..cddf8a2098d 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
@@ -7,6 +7,7 @@
 #include <future>
 
 using search::AttributeVector;
+using search::CompactionStrategy;
 using search::GrowStrategy;
 using search::SerialNum;
 using vespa::config::search::AttributesConfig;
@@ -135,7 +136,7 @@ AttributeCollectionSpec::UP
 AttributeManagerInitializer::createAttributeSpec() const
 {
     uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them
-    AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _fastAccessAttributesOnly);
+    AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _compaction_strategy, _fastAccessAttributesOnly);
     return factory.create(_attrCfg, docIdLimit, _configSerialNum);
 }
 
@@ -146,6 +147,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN
                                                          const AttributesConfig &attrCfg,
                                                          const GrowStrategy &attributeGrow,
                                                          size_t attributeGrowNumDocs,
+                                                         const CompactionStrategy& compaction_strategy,
                                                          bool fastAccessAttributesOnly,
                                                          searchcorespi::index::IThreadService &master,
                                                          std::shared_ptr<AttributeManager::SP> attrMgrResult)
@@ -155,6 +157,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN
       _attrCfg(attrCfg),
       _attributeGrow(attributeGrow),
       _attributeGrowNumDocs(attributeGrowNumDocs),
+      _compaction_strategy(compaction_strategy),
       _fastAccessAttributesOnly(fastAccessAttributesOnly),
       _master(master),
       _attributesResult(),
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
index d56f4c25ace..7c8c34a0d4d 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
@@ -4,6 +4,7 @@
 
 #include "attributemanager.h"
 #include "initialized_attributes_result.h"
+#include <vespa/searchcommon/common/compaction_strategy.h>
 #include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
 #include <vespa/searchcore/proton/initializer/initializer_task.h>
@@ -26,6 +27,7 @@ private:
     vespa::config::search::AttributesConfig _attrCfg;
     search::GrowStrategy _attributeGrow;
     size_t _attributeGrowNumDocs;
+    search::CompactionStrategy _compaction_strategy;
     bool _fastAccessAttributesOnly;
     searchcorespi::index::IThreadService &_master;
     InitializedAttributesResult _attributesResult;
@@ -41,6 +43,7 @@ public:
                                 const vespa::config::search::AttributesConfig &attrCfg,
                                 const search::GrowStrategy &attributeGrow,
                                 size_t attributeGrowNumDocs,
+                                const search::CompactionStrategy& compaction_strategy,
                                 bool fastAccessAttributesOnly,
                                 searchcorespi::index::IThreadService &master,
                                 std::shared_ptr<AttributeManager::SP> attrMgrResult);
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 60878c2b314..0c5b4ef0d33 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -77,13 +77,23 @@ makeGrowStrategy(uint32_t docsInitialCapacity, const Allocation &allocCfg)
     return GrowStrategy(docsInitialCapacity, allocCfg.growfactor, allocCfg.growbias, allocCfg.multivaluegrowfactor);
 }
 
+
+CompactionStrategy
+make_compaction_strategy(const Allocation& alloc_cfg)
+{
+    double max_dead_bytes_ratio = alloc_cfg.maxDeadBytesRatio;
+    double max_dead_address_space_ratio = alloc_cfg.maxDeadAddressSpaceRatio;
+    return CompactionStrategy(max_dead_bytes_ratio, max_dead_address_space_ratio);
+}
+
 DocumentSubDBCollection::Config
 makeSubDBConfig(const ProtonConfig::Distribution & distCfg, const Allocation & allocCfg, size_t numSearcherThreads) {
     size_t initialNumDocs(allocCfg.initialnumdocs);
     GrowStrategy searchableGrowth = makeGrowStrategy(initialNumDocs * distCfg.searchablecopies, allocCfg);
     GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024ul, initialNumDocs/100), allocCfg);
     GrowStrategy notReadyGrowth = makeGrowStrategy(initialNumDocs * (distCfg.redundancy - distCfg.searchablecopies), allocCfg);
-    return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, numSearcherThreads);
+    CompactionStrategy compaction_strategy = make_compaction_strategy(allocCfg);
+    return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, compaction_strategy, numSearcherThreads);
 }
 
 index::IndexConfig
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 61cf45a81d7..52e5858f8a6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -20,10 +20,13 @@ using vespalib::makeLambdaTask;
 namespace proton {
 
 DocumentSubDBCollection::Config::Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed,
-                                        size_t fixedAttributeTotalSkew, size_t numSearchThreads)
+                                        size_t fixedAttributeTotalSkew,
+                                        CompactionStrategy compaction_strategy,
+                                        size_t numSearchThreads)
     : _readyGrowth(ready),
       _notReadyGrowth(notReady),
       _removedGrowth(removed),
+      _compaction_strategy(compaction_strategy),
       _fixedAttributeTotalSkew(fixedAttributeTotalSkew),
       _numSearchThreads(numSearchThreads)
 { }
@@ -66,7 +69,7 @@ DocumentSubDBCollection::DocumentSubDBCollection(
                 SearchableDocSubDB::Config(
                     FastAccessDocSubDB::Config(
                             StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir,
-                                    cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(),
+                                    cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(),
                                     _readySubDbId, SubDbType::READY),
                             true, true, false),
                     cfg.getNumSearchThreads()),
@@ -77,14 +80,14 @@ DocumentSubDBCollection::DocumentSubDBCollection(
     _subDBs.push_back
         (new StoreOnlyDocSubDB(
                 StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, cfg.getRemovedGrowth(),
-                        cfg.getFixedAttributeTotalSkew(), _remSubDbId, SubDbType::REMOVED),
+                        cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _remSubDbId, SubDbType::REMOVED),
                 context));
 
     _subDBs.push_back
         (new FastAccessDocSubDB(
                 FastAccessDocSubDB::Config(
                         StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir,
-                                cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(),
+                                cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(),
                                 _notReadySubDbId, SubDbType::NOTREADY),
                         true, true, true),
                 FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService)));
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index 317ec191d60..c64d0f1a6a8 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -5,6 +5,7 @@
 #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
+#include <vespa/searchcommon/common/compaction_strategy.h>
 #include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/vespalib/util/varholder.h>
@@ -61,18 +62,23 @@ public:
     using SerialNum = search::SerialNum;
     class Config {
     public:
+        using CompactionStrategy = search::CompactionStrategy;
         using GrowStrategy = search::GrowStrategy;
         Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed,
-               size_t fixedAttributeTotalSkew, size_t numSearchThreads);
+               size_t fixedAttributeTotalSkew,
+               CompactionStrategy compaction_strategy,
+               size_t numSearchThreads);
         GrowStrategy getReadyGrowth() const { return _readyGrowth; }
         GrowStrategy getNotReadyGrowth() const { return _notReadyGrowth; }
         GrowStrategy getRemovedGrowth() const { return _removedGrowth; }
+        CompactionStrategy get_compaction_strategy() const { return _compaction_strategy; }
         size_t getNumSearchThreads() const { return _numSearchThreads; }
         size_t getFixedAttributeTotalSkew() const { return _fixedAttributeTotalSkew; }
     private:
         const GrowStrategy _readyGrowth;
         const GrowStrategy _notReadyGrowth;
         const GrowStrategy _removedGrowth;
+        const CompactionStrategy _compaction_strategy;
         const size_t       _fixedAttributeTotalSkew;
         const size_t       _numSearchThreads;
     };
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index f92f3e74e31..5bc6437b4c6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -79,6 +79,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co
                                                          (_hasAttributes ? configSnapshot.getAttributesConfig() : AttributesConfig()),
                                                          _attributeGrow,
                                                          _attributeGrowNumDocs,
+                                                         _attribute_compaction_strategy,
                                                          _fastAccessAttributesOnly,
                                                          _writeService.master(),
                                                          attrMgrResult);
@@ -105,7 +106,7 @@ FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, SerialN
 {
     uint32_t docIdLimit(_dms->getCommittedDocIdLimit());
     AttributeCollectionSpecFactory factory(_attributeGrow,
-            _attributeGrowNumDocs, _fastAccessAttributesOnly);
+            _attributeGrowNumDocs, _attribute_compaction_strategy, _fastAccessAttributesOnly);
     return factory.create(attrCfg, docIdLimit, serialNum);
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index 2aeece204fb..2da0645aef1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -28,6 +28,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.storeonlydocsubdb");
 
+using search::CompactionStrategy;
 using search::GrowStrategy;
 using search::AttributeGuard;
 using search::AttributeVector;
@@ -57,12 +58,14 @@ IIndexWriter::SP nullIndexWriter;
 StoreOnlyDocSubDB::Config::Config(const DocTypeName &docTypeName, const vespalib::string &subName,
                                   const vespalib::string &baseDir,
                                   const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs,
+                                  const search::CompactionStrategy& attribute_compaction_strategy,
                                   uint32_t subDbId, SubDbType subDbType)
     : _docTypeName(docTypeName),
       _subName(subName),
       _baseDir(baseDir + "/" + subName),
       _attributeGrow(attributeGrow),
       _attributeGrowNumDocs(attributeGrowNumDocs),
+      _attribute_compaction_strategy(attribute_compaction_strategy),
       _subDbId(subDbId),
       _subDbType(subDbType)
 { }
@@ -101,6 +104,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx)
       _metaStoreCtx(),
       _attributeGrow(cfg._attributeGrow),
       _attributeGrowNumDocs(cfg._attributeGrowNumDocs),
+      _attribute_compaction_strategy(cfg._attribute_compaction_strategy),
       _flushedDocumentMetaStoreSerialNum(0u),
       _flushedDocumentStoreSerialNum(0u),
       _dms(),
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index 7c3f7c82eb0..a0b278eb491 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -87,12 +87,15 @@ public:
         const vespalib::string _baseDir;
         const search::GrowStrategy _attributeGrow;
         const size_t _attributeGrowNumDocs;
+        const search::CompactionStrategy _attribute_compaction_strategy;
         const uint32_t _subDbId;
         const SubDbType _subDbType;
 
         Config(const DocTypeName &docTypeName, const vespalib::string &subName,
-               const vespalib::string &baseDir, const search::GrowStrategy &attributeGrow,
-               size_t attributeGrowNumDocs, uint32_t subDbId, SubDbType subDbType);
+               const vespalib::string &baseDir,
+               const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs,
+               const search::CompactionStrategy& attribute_compaction_strategy,
+               uint32_t subDbId, SubDbType subDbType);
         ~Config();
     };
 
@@ -132,6 +135,7 @@ protected:
     IDocumentMetaStoreContext::SP _metaStoreCtx;
     const search::GrowStrategy    _attributeGrow;
     const size_t                  _attributeGrowNumDocs;
+    const search::CompactionStrategy _attribute_compaction_strategy;
     // The following two serial numbers reflect state at program startup
     // and are used by replay logic.
     SerialNum                     _flushedDocumentMetaStoreSerialNum;