diff options
Diffstat (limited to 'searchcore/src')
14 files changed, 61 insertions, 13 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index d25a234c6f8..d07c25c2a9e 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -234,6 +234,7 @@ struct ParallelAttributeManager DocumentMetaStore::SP documentMetaStore; search::GrowStrategy attributeGrow; size_t attributeGrowNumDocs; + search::CompactionStrategy attribute_compaction_strategy; bool fastAccessAttributesOnly; std::shared_ptr<AttributeManager::SP> mgr; vespalib::ThreadStackExecutor masterExecutor; @@ -252,13 +253,14 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)), attributeGrow(), attributeGrowNumDocs(1), + attribute_compaction_strategy(), fastAccessAttributesOnly(false), mgr(std::make_shared<AttributeManager::SP>()), masterExecutor(1, 128 * 1024), master(masterExecutor), initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg, - attributeGrow, attributeGrowNumDocs, + attributeGrow, attributeGrowNumDocs, attribute_compaction_strategy, fastAccessAttributesOnly, master, mgr)) { documentMetaStore->setCommittedDocIdLimit(docIdLimit); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index b67eaec1a8d..a9829f3e5f7 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -535,7 +535,7 @@ public: AttributeCollectionSpecFactory _factory; AttributeCollectionSpecTest(bool fastAccessOnly) : _builder(), - _factory(search::GrowStrategy(), 100, fastAccessOnly) + _factory(search::GrowStrategy(), 100, search::CompactionStrategy(), fastAccessOnly) { addAttribute("a1", false); addAttribute("a2", true); diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 9cdd984b152..64178303b40 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -124,7 +124,7 @@ struct MyStoreOnlyConfig SUB_NAME, BASE_DIR, search::GrowStrategy(), - 0, 0, SubDbType::READY) + 0, search::CompactionStrategy(), 0, SubDbType::READY) { } }; diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 17d0ef8ad37..74ec5c2fbef 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -306,6 +306,12 @@ documentdb[].allocation.amortizecount int default=10000 ## used in multi-value attribute vectors to store underlying values. documentdb[].allocation.multivaluegrowfactor double default=0.2 +## The ratio of used bytes that can be dead before attempting to perform compaction. +documentdb[].allocation.max_dead_bytes_ratio double default=0.2 restart + +## The ratio of used address space that can be dead before attempting to perform compaction. +documentdb[].allocation.max_dead_address_space_ratio double default=0.2 restart + ## The interval of when periodic tasks should be run periodic.interval double default=3600.0 diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp index 23701bfdd5d..96db6205bac 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp @@ -11,9 +11,11 @@ namespace proton { AttributeCollectionSpecFactory::AttributeCollectionSpecFactory( const search::GrowStrategy &growStrategy, size_t growNumDocs, + const search::CompactionStrategy& compaction_strategy, bool fastAccessOnly) : _growStrategy(growStrategy), _growNumDocs(growNumDocs), + _compaction_strategy(compaction_strategy), _fastAccessOnly(fastAccessOnly) { } @@ -35,6 +37,7 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, } grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew); cfg.setGrowStrategy(grow); + cfg.setCompactionStrategy(_compaction_strategy); attrs.push_back(AttributeSpec(attr.name, cfg)); } return std::make_unique<AttributeCollectionSpec>(attrs, docIdLimit, serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h index bfa6681d6f2..9e877bedf28 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h @@ -4,6 +4,7 @@ #include "attribute_collection_spec.h" #include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/config-attributes.h> @@ -21,11 +22,13 @@ private: const search::GrowStrategy _growStrategy; const size_t _growNumDocs; + const search::CompactionStrategy _compaction_strategy; const bool _fastAccessOnly; public: AttributeCollectionSpecFactory(const search::GrowStrategy &growStrategy, size_t growNumDocs, + const search::CompactionStrategy& compaction_strategy, bool fastAccessOnly); AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index 9198cfdafab..cddf8a2098d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -7,6 +7,7 @@ #include <future> using search::AttributeVector; +using search::CompactionStrategy; using search::GrowStrategy; using search::SerialNum; using vespa::config::search::AttributesConfig; @@ -135,7 +136,7 @@ AttributeCollectionSpec::UP AttributeManagerInitializer::createAttributeSpec() const { uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them - AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _fastAccessAttributesOnly); + AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _compaction_strategy, _fastAccessAttributesOnly); return factory.create(_attrCfg, docIdLimit, _configSerialNum); } @@ -146,6 +147,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN const AttributesConfig &attrCfg, const GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, + const CompactionStrategy& compaction_strategy, bool fastAccessAttributesOnly, searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult) @@ -155,6 +157,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN _attrCfg(attrCfg), _attributeGrow(attributeGrow), _attributeGrowNumDocs(attributeGrowNumDocs), + _compaction_strategy(compaction_strategy), _fastAccessAttributesOnly(fastAccessAttributesOnly), _master(master), _attributesResult(), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h index d56f4c25ace..7c8c34a0d4d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h @@ -4,6 +4,7 @@ #include "attributemanager.h" #include "initialized_attributes_result.h" +#include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> @@ -26,6 +27,7 @@ private: vespa::config::search::AttributesConfig _attrCfg; search::GrowStrategy _attributeGrow; size_t _attributeGrowNumDocs; + search::CompactionStrategy _compaction_strategy; bool _fastAccessAttributesOnly; searchcorespi::index::IThreadService &_master; InitializedAttributesResult _attributesResult; @@ -41,6 +43,7 @@ public: const vespa::config::search::AttributesConfig &attrCfg, const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, + const search::CompactionStrategy& compaction_strategy, bool fastAccessAttributesOnly, searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 60878c2b314..0c5b4ef0d33 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -77,13 +77,23 @@ makeGrowStrategy(uint32_t docsInitialCapacity, const Allocation &allocCfg) return GrowStrategy(docsInitialCapacity, allocCfg.growfactor, allocCfg.growbias, allocCfg.multivaluegrowfactor); } + +CompactionStrategy +make_compaction_strategy(const Allocation& alloc_cfg) +{ + double max_dead_bytes_ratio = alloc_cfg.maxDeadBytesRatio; + double max_dead_address_space_ratio = alloc_cfg.maxDeadAddressSpaceRatio; + return CompactionStrategy(max_dead_bytes_ratio, max_dead_address_space_ratio); +} + DocumentSubDBCollection::Config makeSubDBConfig(const ProtonConfig::Distribution & distCfg, const Allocation & allocCfg, size_t numSearcherThreads) { size_t initialNumDocs(allocCfg.initialnumdocs); GrowStrategy searchableGrowth = makeGrowStrategy(initialNumDocs * distCfg.searchablecopies, allocCfg); GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024ul, initialNumDocs/100), allocCfg); GrowStrategy notReadyGrowth = makeGrowStrategy(initialNumDocs * (distCfg.redundancy - distCfg.searchablecopies), allocCfg); - return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, numSearcherThreads); + CompactionStrategy compaction_strategy = make_compaction_strategy(allocCfg); + return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, compaction_strategy, numSearcherThreads); } index::IndexConfig diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 61cf45a81d7..52e5858f8a6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -20,10 +20,13 @@ using vespalib::makeLambdaTask; namespace proton { DocumentSubDBCollection::Config::Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed, - size_t fixedAttributeTotalSkew, size_t numSearchThreads) + size_t fixedAttributeTotalSkew, + CompactionStrategy compaction_strategy, + size_t numSearchThreads) : _readyGrowth(ready), _notReadyGrowth(notReady), _removedGrowth(removed), + _compaction_strategy(compaction_strategy), _fixedAttributeTotalSkew(fixedAttributeTotalSkew), _numSearchThreads(numSearchThreads) { } @@ -66,7 +69,7 @@ DocumentSubDBCollection::DocumentSubDBCollection( SearchableDocSubDB::Config( FastAccessDocSubDB::Config( StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir, - cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(), + cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _readySubDbId, SubDbType::READY), true, true, false), cfg.getNumSearchThreads()), @@ -77,14 +80,14 @@ DocumentSubDBCollection::DocumentSubDBCollection( _subDBs.push_back (new StoreOnlyDocSubDB( StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, cfg.getRemovedGrowth(), - cfg.getFixedAttributeTotalSkew(), _remSubDbId, SubDbType::REMOVED), + cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _remSubDbId, SubDbType::REMOVED), context)); _subDBs.push_back (new FastAccessDocSubDB( FastAccessDocSubDB::Config( StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir, - cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(), + cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _notReadySubDbId, SubDbType::NOTREADY), true, true, true), FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService))); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 317ec191d60..c64d0f1a6a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -5,6 +5,7 @@ #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> +#include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/util/varholder.h> @@ -61,18 +62,23 @@ public: using SerialNum = search::SerialNum; class Config { public: + using CompactionStrategy = search::CompactionStrategy; using GrowStrategy = search::GrowStrategy; Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed, - size_t fixedAttributeTotalSkew, size_t numSearchThreads); + size_t fixedAttributeTotalSkew, + CompactionStrategy compaction_strategy, + size_t numSearchThreads); GrowStrategy getReadyGrowth() const { return _readyGrowth; } GrowStrategy getNotReadyGrowth() const { return _notReadyGrowth; } GrowStrategy getRemovedGrowth() const { return _removedGrowth; } + CompactionStrategy get_compaction_strategy() const { return _compaction_strategy; } size_t getNumSearchThreads() const { return _numSearchThreads; } size_t getFixedAttributeTotalSkew() const { return _fixedAttributeTotalSkew; } private: const GrowStrategy _readyGrowth; const GrowStrategy _notReadyGrowth; const GrowStrategy _removedGrowth; + const CompactionStrategy _compaction_strategy; const size_t _fixedAttributeTotalSkew; const size_t _numSearchThreads; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index f92f3e74e31..5bc6437b4c6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -79,6 +79,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co (_hasAttributes ? configSnapshot.getAttributesConfig() : AttributesConfig()), _attributeGrow, _attributeGrowNumDocs, + _attribute_compaction_strategy, _fastAccessAttributesOnly, _writeService.master(), attrMgrResult); @@ -105,7 +106,7 @@ FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, SerialN { uint32_t docIdLimit(_dms->getCommittedDocIdLimit()); AttributeCollectionSpecFactory factory(_attributeGrow, - _attributeGrowNumDocs, _fastAccessAttributesOnly); + _attributeGrowNumDocs, _attribute_compaction_strategy, _fastAccessAttributesOnly); return factory.create(attrCfg, docIdLimit, serialNum); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 2aeece204fb..2da0645aef1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -28,6 +28,7 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.server.storeonlydocsubdb"); +using search::CompactionStrategy; using search::GrowStrategy; using search::AttributeGuard; using search::AttributeVector; @@ -57,12 +58,14 @@ IIndexWriter::SP nullIndexWriter; StoreOnlyDocSubDB::Config::Config(const DocTypeName &docTypeName, const vespalib::string &subName, const vespalib::string &baseDir, const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, + const search::CompactionStrategy& attribute_compaction_strategy, uint32_t subDbId, SubDbType subDbType) : _docTypeName(docTypeName), _subName(subName), _baseDir(baseDir + "/" + subName), _attributeGrow(attributeGrow), _attributeGrowNumDocs(attributeGrowNumDocs), + _attribute_compaction_strategy(attribute_compaction_strategy), _subDbId(subDbId), _subDbType(subDbType) { } @@ -101,6 +104,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _metaStoreCtx(), _attributeGrow(cfg._attributeGrow), _attributeGrowNumDocs(cfg._attributeGrowNumDocs), + _attribute_compaction_strategy(cfg._attribute_compaction_strategy), _flushedDocumentMetaStoreSerialNum(0u), _flushedDocumentStoreSerialNum(0u), _dms(), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 7c3f7c82eb0..a0b278eb491 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -87,12 +87,15 @@ public: const vespalib::string _baseDir; const search::GrowStrategy _attributeGrow; const size_t _attributeGrowNumDocs; + const search::CompactionStrategy _attribute_compaction_strategy; const uint32_t _subDbId; const SubDbType _subDbType; Config(const DocTypeName &docTypeName, const vespalib::string &subName, - const vespalib::string &baseDir, const search::GrowStrategy &attributeGrow, - size_t attributeGrowNumDocs, uint32_t subDbId, SubDbType subDbType); + const vespalib::string &baseDir, + const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, + const search::CompactionStrategy& attribute_compaction_strategy, + uint32_t subDbId, SubDbType subDbType); ~Config(); }; @@ -132,6 +135,7 @@ protected: IDocumentMetaStoreContext::SP _metaStoreCtx; const search::GrowStrategy _attributeGrow; const size_t _attributeGrowNumDocs; + const search::CompactionStrategy _attribute_compaction_strategy; // The following two serial numbers reflect state at program startup // and are used by replay logic. SerialNum _flushedDocumentMetaStoreSerialNum; |