diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2021-02-02 15:34:28 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@broadpark.no> | 2021-02-02 15:34:28 +0100 |
commit | 3a4df233f6e4815ac356509ddfa0bc9076bb0376 (patch) | |
tree | 3ebc907d6f1197ca52e4e9505f7a00896864b3f9 | |
parent | 4b3b7543f0f8acf3f72d25d76e36d63841de278a (diff) |
Improve handling of changed alloc config.
41 files changed, 432 insertions, 157 deletions
diff --git a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt index 23e6e8dd394..bf5686280e6 100644 --- a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt @@ -1,7 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcommon_searchcommon_common OBJECT SOURCES + compaction_strategy.cpp datatype.cpp + growstrategy.cpp schema.cpp schemaconfigurer.cpp DEPENDS diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp new file mode 100644 index 00000000000..c3377ed5857 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "compaction_strategy.h" +#include <iostream> +namespace search { + +std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy) +{ + os << "{maxDeadBytesRatio=" << compaction_strategy.getMaxDeadBytesRatio() << + ", maxDeadAddressSpaceRatio=" << compaction_strategy.getMaxDeadAddressSpaceRatio() << + "}"; + return os; +} + +} diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h index 0b3caf44481..ae354a4c4d2 100644 --- a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h +++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h @@ -3,6 +3,7 @@ #pragma once #include <stdint.h> +#include <iosfwd> namespace search { @@ -34,4 +35,6 @@ public: bool operator!=(const CompactionStrategy & rhs) const { return !(operator==(rhs)); } }; +std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy); + } // namespace search diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp b/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp new file mode 100644 index 00000000000..534be3060a7 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "growstrategy.h" +#include <iostream> + +namespace search { + +std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy) +{ + os << "{docsInitialCapacity=" << grow_strategy.getDocsInitialCapacity() << + ", docsGrowFactor=" << grow_strategy.getDocsGrowFactor() << + ", docsGrowDelta=" << grow_strategy.getDocsGrowDelta() << + ", multiValueAllocGrowFactor=" << grow_strategy.getMultiValueAllocGrowFactor() << + "}"; + return os; +} + +} diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.h b/searchcommon/src/vespa/searchcommon/common/growstrategy.h index 13220741e51..f7c5c030d95 100644 --- a/searchcommon/src/vespa/searchcommon/common/growstrategy.h +++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/growstrategy.h> #include <cstdint> +#include <iosfwd> namespace search { @@ -54,5 +55,7 @@ public: } }; +std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy); + } diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 04cface5d40..d77bc69ba62 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -69,6 +69,7 @@ vespa_define_module( src/tests/proton/attribute/imported_attributes_repo src/tests/proton/bucketdb/bucketdb src/tests/proton/common + src/tests/proton/common/alloc_config src/tests/proton/common/attribute_updater src/tests/proton/common/document_type_inspector src/tests/proton/common/hw_info_sampler diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 4715ff80d03..b47a1954c6f 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -13,6 +13,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> @@ -145,6 +146,7 @@ public: std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const AllocConfig>(), "client", docTypeName.getName()); } diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 90942e9aef4..6cca2e4bd48 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -38,6 +38,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/metrics/config-metricsmanager.h> #include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> @@ -198,6 +199,7 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const AllocConfig>(), "client", doc_type_name.getName()); } diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index d07c25c2a9e..3987a8685ea 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -232,9 +232,7 @@ struct ParallelAttributeManager InitializerTask::SP documentMetaStoreInitTask; BucketDBOwner::SP bucketDbOwner; DocumentMetaStore::SP documentMetaStore; - search::GrowStrategy attributeGrow; - size_t attributeGrowNumDocs; - search::CompactionStrategy attribute_compaction_strategy; + AllocStrategy alloc_strategy; bool fastAccessAttributesOnly; std::shared_ptr<AttributeManager::SP> mgr; vespalib::ThreadStackExecutor masterExecutor; @@ -251,16 +249,14 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria : documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()), bucketDbOwner(std::make_shared<BucketDBOwner>()), documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)), - attributeGrow(), - attributeGrowNumDocs(1), - attribute_compaction_strategy(), + alloc_strategy(), fastAccessAttributesOnly(false), mgr(std::make_shared<AttributeManager::SP>()), masterExecutor(1, 128 * 1024), master(masterExecutor), initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg, - attributeGrow, attributeGrowNumDocs, attribute_compaction_strategy, + alloc_strategy, fastAccessAttributesOnly, master, mgr)) { documentMetaStore->setCommittedDocIdLimit(docIdLimit); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index a9829f3e5f7..ebd3e27aeca 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -535,7 +535,7 @@ public: AttributeCollectionSpecFactory _factory; AttributeCollectionSpecTest(bool fastAccessOnly) : _builder(), - _factory(search::GrowStrategy(), 100, search::CompactionStrategy(), fastAccessOnly) + _factory(AllocStrategy(search::GrowStrategy(), search::CompactionStrategy(), 100), fastAccessOnly) { addAttribute("a1", false); addAttribute("a2", true); diff --git a/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt b/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt new file mode 100644 index 00000000000..26a2dc72cc2 --- /dev/null +++ b/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_alloc_config_test_app TEST + SOURCES + alloc_config_test.cpp + DEPENDS + searchcore_pcommon + GTest::GTest +) +vespa_add_test(NAME searchcore_alloc_config_test_app COMMAND searchcore_alloc_config_test_app) diff --git a/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp new file mode 100644 index 00000000000..2c2e03dcfd0 --- /dev/null +++ b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/common/alloc_config.h> +#include <vespa/searchcore/proton/common/subdbtype.h> +#include <vespa/vespalib/gtest/gtest.h> + +using proton::AllocConfig; +using proton::AllocStrategy; +using proton::SubDbType; +using search::CompactionStrategy; +using search::GrowStrategy; + +namespace { + +CompactionStrategy baseline_compaction_strategy(0.2, 0.25); + +GrowStrategy make_grow_strategy(uint32_t initial_docs) { + return GrowStrategy(initial_docs, 0.1, 1, 0.15); +} + +AllocStrategy make_alloc_strategy(uint32_t initial_docs) { + return AllocStrategy(make_grow_strategy(initial_docs), baseline_compaction_strategy, 10000); +} + +}; + +TEST(AllocConfigTest, hello_world) +{ + AllocConfig config(make_alloc_strategy(10000000), 5, 2); + EXPECT_EQ(make_alloc_strategy(20000000), config.make_alloc_strategy(SubDbType::READY)); + EXPECT_EQ(make_alloc_strategy(100000), config.make_alloc_strategy(SubDbType::REMOVED)); + EXPECT_EQ(make_alloc_strategy(30000000), config.make_alloc_strategy(SubDbType::NOTREADY)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 64178303b40..40a9656ae41 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -123,8 +123,7 @@ struct MyStoreOnlyConfig : _cfg(DocTypeName(DOCTYPE_NAME), SUB_NAME, BASE_DIR, - search::GrowStrategy(), - 0, search::CompactionStrategy(), 0, SubDbType::READY) + 0, SubDbType::READY) { } }; diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 5a0cfc18c78..4f8e8e8aa8c 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -9,6 +9,7 @@ #include <vespa/config-summarymap.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> @@ -103,6 +104,7 @@ struct DBConfigFixture { std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const AllocConfig>(), configId, docTypeName); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp index 96db6205bac..4189688ea81 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp @@ -9,13 +9,9 @@ using search::GrowStrategy; namespace proton { AttributeCollectionSpecFactory::AttributeCollectionSpecFactory( - const search::GrowStrategy &growStrategy, - size_t growNumDocs, - const search::CompactionStrategy& compaction_strategy, + const AllocStrategy &alloc_strategy, bool fastAccessOnly) - : _growStrategy(growStrategy), - _growNumDocs(growNumDocs), - _compaction_strategy(compaction_strategy), + : _alloc_strategy(alloc_strategy), _fastAccessOnly(fastAccessOnly) { } @@ -27,8 +23,8 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, { AttributeCollectionSpec::AttributeList attrs; // Amortize memory spike cost over N docs - const size_t skew = _growNumDocs/(attrCfg.attribute.size()+1); - GrowStrategy grow = _growStrategy; + const size_t skew = _alloc_strategy.get_amortize_count()/(attrCfg.attribute.size()+1); + GrowStrategy grow = _alloc_strategy.get_grow_strategy(); grow.setDocsInitialCapacity(std::max(grow.getDocsInitialCapacity(),docIdLimit)); for (const auto &attr : attrCfg.attribute) { search::attribute::Config cfg = ConfigConverter::convert(attr); @@ -37,7 +33,7 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, } grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew); cfg.setGrowStrategy(grow); - cfg.setCompactionStrategy(_compaction_strategy); + cfg.setCompactionStrategy(_alloc_strategy.get_compaction_strategy()); attrs.push_back(AttributeSpec(attr.name, cfg)); } return std::make_unique<AttributeCollectionSpec>(attrs, docIdLimit, serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h index 9e877bedf28..074c56448f3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h @@ -4,8 +4,7 @@ #include "attribute_collection_spec.h" #include <vespa/searchcommon/attribute/config.h> -#include <vespa/searchcommon/common/compaction_strategy.h> -#include <vespa/searchcommon/common/growstrategy.h> +#include <vespa/searchcore/proton/common/alloc_strategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/config-attributes.h> @@ -20,15 +19,11 @@ class AttributeCollectionSpecFactory private: typedef vespa::config::search::AttributesConfig AttributesConfig; - const search::GrowStrategy _growStrategy; - const size_t _growNumDocs; - const search::CompactionStrategy _compaction_strategy; + const AllocStrategy _alloc_strategy; const bool _fastAccessOnly; public: - AttributeCollectionSpecFactory(const search::GrowStrategy &growStrategy, - size_t growNumDocs, - const search::CompactionStrategy& compaction_strategy, + AttributeCollectionSpecFactory(const AllocStrategy& alloc_strategy, bool fastAccessOnly); AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index cddf8a2098d..109a4bb2192 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -136,7 +136,7 @@ AttributeCollectionSpec::UP AttributeManagerInitializer::createAttributeSpec() const { uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them - AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _compaction_strategy, _fastAccessAttributesOnly); + AttributeCollectionSpecFactory factory(_alloc_strategy, _fastAccessAttributesOnly); return factory.create(_attrCfg, docIdLimit, _configSerialNum); } @@ -145,9 +145,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN DocumentMetaStore::SP documentMetaStore, AttributeManager::SP baseAttrMgr, const AttributesConfig &attrCfg, - const GrowStrategy &attributeGrow, - size_t attributeGrowNumDocs, - const CompactionStrategy& compaction_strategy, + const AllocStrategy& alloc_strategy, bool fastAccessAttributesOnly, searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult) @@ -155,9 +153,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN _documentMetaStore(documentMetaStore), _attrMgr(), _attrCfg(attrCfg), - _attributeGrow(attributeGrow), - _attributeGrowNumDocs(attributeGrowNumDocs), - _compaction_strategy(compaction_strategy), + _alloc_strategy(alloc_strategy), _fastAccessAttributesOnly(fastAccessAttributesOnly), _master(master), _attributesResult(), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h index 7c8c34a0d4d..f74a0f1519d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h @@ -4,10 +4,9 @@ #include "attributemanager.h" #include "initialized_attributes_result.h" -#include <vespa/searchcommon/common/compaction_strategy.h> -#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> +#include <vespa/searchcore/proton/common/alloc_strategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/config-attributes.h> @@ -25,9 +24,7 @@ private: DocumentMetaStore::SP _documentMetaStore; AttributeManager::SP _attrMgr; vespa::config::search::AttributesConfig _attrCfg; - search::GrowStrategy _attributeGrow; - size_t _attributeGrowNumDocs; - search::CompactionStrategy _compaction_strategy; + AllocStrategy _alloc_strategy; bool _fastAccessAttributesOnly; searchcorespi::index::IThreadService &_master; InitializedAttributesResult _attributesResult; @@ -41,9 +38,7 @@ public: DocumentMetaStore::SP documentMetaStore, AttributeManager::SP baseAttrMgr, const vespa::config::search::AttributesConfig &attrCfg, - const search::GrowStrategy &attributeGrow, - size_t attributeGrowNumDocs, - const search::CompactionStrategy& compaction_strategy, + const AllocStrategy& alloc_strategy, bool fastAccessAttributesOnly, searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 0b00e274522..f635ee34a04 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -177,10 +177,14 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, auto shrinker = wrap->getShrinker(); assert(shrinker); addAttribute(AttributeWrap::normalAttribute(av), shrinker); + auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix()); + auto cfg = aspec.getConfig(); + _attributeFieldWriter.execute(id, [av, cfg]() { av->update_config(cfg); }); } else { toBeAdded.push_back(aspec); } } + _attributeFieldWriter.sync(); } void diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index d59a4075dce..a91c35f0485 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -1,6 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcore_pcommon STATIC SOURCES + alloc_config.cpp + alloc_strategy.cpp attribute_updater.cpp attributefieldvaluenode.cpp cachedselect.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp new file mode 100644 index 00000000000..1611d00fb0f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp @@ -0,0 +1,56 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "alloc_config.h" +#include <vespa/searchcore/proton/common/subdbtype.h> +#include <algorithm> + +using search::CompactionStrategy; +using search::GrowStrategy; + +namespace proton { + +AllocConfig::AllocConfig(const AllocStrategy& alloc_strategy, + uint32_t redundancy, uint32_t searchable_copies) + : _alloc_strategy(alloc_strategy), + _redundancy(redundancy), + _searchable_copies(searchable_copies) +{ +} + +AllocConfig::AllocConfig() + : AllocConfig(AllocStrategy(), 1, 1) +{ +} + +AllocConfig::~AllocConfig() = default; + +bool +AllocConfig::operator==(const AllocConfig &rhs) const noexcept +{ + return ((_alloc_strategy == rhs._alloc_strategy) && + (_redundancy == rhs._redundancy) && + (_searchable_copies == rhs._searchable_copies)); +} + +AllocStrategy +AllocConfig::make_alloc_strategy(SubDbType sub_db_type) const +{ + auto &baseline_grow_strategy = _alloc_strategy.get_grow_strategy(); + size_t initial_capacity = baseline_grow_strategy.getDocsInitialCapacity(); + switch (sub_db_type) { + case SubDbType::READY: + initial_capacity *= _searchable_copies; + break; + case SubDbType::NOTREADY: + initial_capacity *= (_redundancy - _searchable_copies); + break; + case SubDbType::REMOVED: + default: + initial_capacity = std::max(1024ul, initial_capacity / 100); + break; + } + GrowStrategy grow_strategy(initial_capacity, baseline_grow_strategy.getDocsGrowFactor(), baseline_grow_strategy.getDocsGrowDelta(), baseline_grow_strategy.getMultiValueAllocGrowFactor()); + return AllocStrategy(grow_strategy, _alloc_strategy.get_compaction_strategy(), _alloc_strategy.get_amortize_count()); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.h b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h new file mode 100644 index 00000000000..b9ea7a11dd2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h @@ -0,0 +1,33 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "alloc_strategy.h" + +namespace proton { + +enum class SubDbType; + +/* + * Class representing allocation config for proton which can be used + * to make an allocation strategy. + */ +class AllocConfig +{ + AllocStrategy _alloc_strategy; // baseline before adjusting for redundancy / searchable copies + const uint32_t _redundancy; + const uint32_t _searchable_copies; + +public: + AllocConfig(const AllocStrategy& alloc_strategy, uint32_t redundancy, uint32_t searchable_copies); + AllocConfig(); + ~AllocConfig(); + + bool operator==(const AllocConfig &rhs) const noexcept; + bool operator!=(const AllocConfig &rhs) const noexcept { + return !operator==(rhs); + } + AllocStrategy make_alloc_strategy(SubDbType sub_db_type) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp new file mode 100644 index 00000000000..3af72757ccb --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "alloc_strategy.h" +#include <iostream> + +using search::CompactionStrategy; +using search::GrowStrategy; + +namespace proton { + +AllocStrategy::AllocStrategy(const GrowStrategy& grow_strategy, + const CompactionStrategy& compaction_strategy, + uint32_t amortize_count) + : _grow_strategy(grow_strategy), + _compaction_strategy(compaction_strategy), + _amortize_count(amortize_count) +{ +} + +AllocStrategy::AllocStrategy() + : AllocStrategy(GrowStrategy(), CompactionStrategy(), 10000) +{ +} + +AllocStrategy::~AllocStrategy() = default; + +bool +AllocStrategy::operator==(const AllocStrategy &rhs) const noexcept +{ + return ((_grow_strategy == rhs._grow_strategy) && + (_compaction_strategy == rhs._compaction_strategy) && + (_amortize_count == rhs._amortize_count)); +} + +std::ostream& operator<<(std::ostream& os, const AllocStrategy&alloc_strategy) +{ + os << "{ grow_strategy=" << alloc_strategy.get_grow_strategy() << ", compaction_strategy=" << alloc_strategy.get_compaction_strategy() << ", amortize_count=" << alloc_strategy.get_amortize_count() << "}"; + return os; +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h new file mode 100644 index 00000000000..517c5af8cb4 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/searchcommon/common/growstrategy.h> +#include <iosfwd> + +namespace proton { + +/* + * Class representing allocation strategy + */ +class AllocStrategy +{ +protected: + const search::GrowStrategy _grow_strategy; + const search::CompactionStrategy _compaction_strategy; + const uint32_t _amortize_count; + +public: + AllocStrategy(const search::GrowStrategy& grow_strategy, + const search::CompactionStrategy& compaction_strategy, + uint32_t amortize_count); + + AllocStrategy(); + ~AllocStrategy(); + + bool operator==(const AllocStrategy &rhs) const noexcept; + bool operator!=(const AllocStrategy &rhs) const noexcept { + return !operator==(rhs); + } + const search::GrowStrategy& get_grow_strategy() const noexcept { return _grow_strategy; } + const search::CompactionStrategy& get_compaction_strategy() const noexcept { return _compaction_strategy; } + uint32_t get_amortize_count() const noexcept { return _amortize_count; } +}; + +std::ostream& operator<<(std::ostream& os, const AllocStrategy&alloc_strategy); + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 0c5b4ef0d33..166d1e928bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -70,49 +70,11 @@ namespace proton { namespace { constexpr uint32_t indexing_thread_stack_size = 128 * 1024; -using Allocation = ProtonConfig::Documentdb::Allocation; -GrowStrategy -makeGrowStrategy(uint32_t docsInitialCapacity, const Allocation &allocCfg) -{ - return GrowStrategy(docsInitialCapacity, allocCfg.growfactor, allocCfg.growbias, allocCfg.multivaluegrowfactor); -} - - -CompactionStrategy -make_compaction_strategy(const Allocation& alloc_cfg) -{ - double max_dead_bytes_ratio = alloc_cfg.maxDeadBytesRatio; - double max_dead_address_space_ratio = alloc_cfg.maxDeadAddressSpaceRatio; - return CompactionStrategy(max_dead_bytes_ratio, max_dead_address_space_ratio); -} - -DocumentSubDBCollection::Config -makeSubDBConfig(const ProtonConfig::Distribution & distCfg, const Allocation & allocCfg, size_t numSearcherThreads) { - size_t initialNumDocs(allocCfg.initialnumdocs); - GrowStrategy searchableGrowth = makeGrowStrategy(initialNumDocs * distCfg.searchablecopies, allocCfg); - GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024ul, initialNumDocs/100), allocCfg); - GrowStrategy notReadyGrowth = makeGrowStrategy(initialNumDocs * (distCfg.redundancy - distCfg.searchablecopies), allocCfg); - CompactionStrategy compaction_strategy = make_compaction_strategy(allocCfg); - return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, compaction_strategy, numSearcherThreads); -} - index::IndexConfig makeIndexConfig(const ProtonConfig::Index & cfg) { return index::IndexConfig(WarmupConfig(vespalib::from_s(cfg.warmup.time), cfg.warmup.unpack), cfg.maxflushed, cfg.cache.size); } -ProtonConfig::Documentdb _G_defaultProtonDocumentDBConfig; - -const ProtonConfig::Documentdb * -findDocumentDB(const ProtonConfig::DocumentdbVector & documentDBs, const vespalib::string & docType) { - for (const auto & dbCfg : documentDBs) { - if (dbCfg.inputdoctypename == docType) { - return & dbCfg; - } - } - return &_G_defaultProtonDocumentDBConfig; -} - class MetricsUpdateHook : public metrics::UpdateHook { DocumentDB &_db; public: @@ -193,9 +155,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, - makeSubDBConfig(protonCfg.distribution, - findDocumentDB(protonCfg.documentdb, docTypeName.getName())->allocation, - protonCfg.numsearcherthreads), + DocumentSubDBCollection::Config(protonCfg.numsearcherthreads), hwInfo), _maintenanceController(_writeService.master(), sharedExecutor, _docTypeName), _lidSpaceCompactionHandlers(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index baa6c8eb450..7c487043b5b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/searchcore/proton/attribute/attribute_aspect_delayer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/searchcore/proton/common/indexschema_inspector.h> @@ -49,7 +50,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() storeChanged(false), visibilityDelayChanged(false), flushChanged(false), - threading_service_config_changed(false) + threading_service_config_changed(false), + alloc_config_changed(false) { } DocumentDBConfig::DocumentDBConfig( @@ -70,6 +72,7 @@ DocumentDBConfig::DocumentDBConfig( const DocumentDBMaintenanceConfig::SP &maintenance, const search::LogDocumentStore::Config & storeConfig, std::shared_ptr<const ThreadingServiceConfig> threading_service_config, + std::shared_ptr<const AllocConfig> alloc_config, const vespalib::string &configId, const vespalib::string &docTypeName) noexcept : _configId(configId), @@ -91,6 +94,7 @@ DocumentDBConfig::DocumentDBConfig( _maintenance(maintenance), _storeConfig(storeConfig), _threading_service_config(std::move(threading_service_config)), + _alloc_config(std::move(alloc_config)), _orig(), _delayedAttributeAspects(false) { } @@ -117,6 +121,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _maintenance(cfg._maintenance), _storeConfig(cfg._storeConfig), _threading_service_config(cfg._threading_service_config), + _alloc_config(cfg._alloc_config), _orig(cfg._orig), _delayedAttributeAspects(false) { } @@ -141,7 +146,8 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const equals<Schema>(_schema.get(), rhs._schema.get()) && equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()) && _storeConfig == rhs._storeConfig && - equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()); + equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()) && + equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get()); } @@ -167,6 +173,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.visibilityDelayChanged = (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay()); retval.flushChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get(), [](const auto &l, const auto &r) { return l.getFlushConfig() == r.getFlushConfig(); }); retval.threading_service_config_changed = !equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()); + retval.alloc_config_changed = !equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get()); return retval; } @@ -188,7 +195,8 @@ DocumentDBConfig::valid() const _tuneFileDocumentDB && _schema && _maintenance && - _threading_service_config; + _threading_service_config && + _alloc_config; } namespace @@ -232,6 +240,7 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) o._maintenance, o._storeConfig, o._threading_service_config, + o._alloc_config, o._configId, o._docTypeName); ret->_orig = orig; @@ -274,6 +283,7 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) _maintenance, _storeConfig, _threading_service_config, + _alloc_config, _configId, _docTypeName); } @@ -311,6 +321,7 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum n._maintenance, n._storeConfig, n._threading_service_config, + n._alloc_config, n._configId, n._docTypeName); result->_delayedAttributeAspects = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index dc163e91ade..c6438f6b2fb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -30,6 +30,7 @@ namespace document::internal { class InternalDocumenttypesType; } namespace proton { class ThreadingServiceConfig; +class AllocConfig; class DocumentDBConfig { @@ -55,6 +56,7 @@ public: bool visibilityDelayChanged; bool flushChanged; bool threading_service_config_changed; + bool alloc_config_changed; ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } @@ -131,6 +133,7 @@ private: MaintenanceConfigSP _maintenance; search::LogDocumentStore::Config _storeConfig; std::shared_ptr<const ThreadingServiceConfig> _threading_service_config; + std::shared_ptr<const AllocConfig> _alloc_config; SP _orig; bool _delayedAttributeAspects; @@ -169,6 +172,7 @@ public: const DocumentDBMaintenanceConfig::SP &maintenance, const search::LogDocumentStore::Config & storeConfig, std::shared_ptr<const ThreadingServiceConfig> threading_service_config, + std::shared_ptr<const AllocConfig> alloc_config, const vespalib::string &configId, const vespalib::string &docTypeName) noexcept; @@ -210,6 +214,8 @@ public: bool getDelayedAttributeAspects() const { return _delayedAttributeAspects; } const ThreadingServiceConfig& get_threading_service_config() const { return *_threading_service_config; } const std::shared_ptr<const ThreadingServiceConfig>& get_threading_service_config_shared_ptr() const { return _threading_service_config; } + const AllocConfig& get_alloc_config() const { return *_alloc_config; } + const std::shared_ptr<const AllocConfig>& get_alloc_config_shared_ptr() const { return _alloc_config; } bool operator==(const DocumentDBConfig &rhs) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 115a49fe997..f5a594d2f36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -3,6 +3,7 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" #include "threading_service_config.h" +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/config/config-onnx-models.h> @@ -252,6 +253,19 @@ build_threading_service_config(const ProtonConfig &proton_config, hw_info.cpu())); } +std::shared_ptr<const AllocConfig> +build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& doc_type_name) +{ + auto& document_db_config_entry = find_document_db_config_entry(proton_config.documentdb, doc_type_name); + auto& alloc_config = document_db_config_entry.allocation; + auto& distribution_config = proton_config.distribution; + search::GrowStrategy grow_strategy(alloc_config.initialnumdocs, alloc_config.growfactor, alloc_config.growbias, alloc_config.multivaluegrowfactor); + search::CompactionStrategy compaction_strategy(alloc_config.maxDeadBytesRatio, alloc_config.maxDeadAddressSpaceRatio); + return std::make_shared<const AllocConfig> + (AllocStrategy(grow_strategy, compaction_strategy, alloc_config.amortizecount), + distribution_config.redundancy, distribution_config.searchablecopies); +} + } void @@ -275,6 +289,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) MaintenanceConfigSP oldMaintenanceConfig; MaintenanceConfigSP newMaintenanceConfig; std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config; + std::shared_ptr<const AllocConfig> old_alloc_config; if (!_ignoreForwardedConfig) { if (!(_bootstrapConfig->getDocumenttypesConfigSP() && @@ -299,6 +314,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) newIndexschemaConfig = current->getIndexschemaConfigSP(); oldMaintenanceConfig = current->getMaintenanceConfigSP(); old_threading_service_config = current->get_threading_service_config_shared_ptr(); + old_alloc_config = current->get_alloc_config_shared_ptr(); currentGeneration = current->getGeneration(); } @@ -382,6 +398,10 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) (*new_threading_service_config == *old_threading_service_config)) { new_threading_service_config = old_threading_service_config; } + auto new_alloc_config = build_alloc_config(_bootstrapConfig->getProtonConfig(), _docTypeName); + if (new_alloc_config && old_alloc_config &&(*new_alloc_config == *old_alloc_config)) { + new_alloc_config = old_alloc_config; + } auto newSnapshot = std::make_shared<DocumentDBConfig>(generation, newRankProfilesConfig, newRankingConstants, @@ -399,6 +419,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) newMaintenanceConfig, storeConfig, new_threading_service_config, + new_alloc_config, _configId, _docTypeName); assert(newSnapshot->valid()); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 52e5858f8a6..6ce0b896a50 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -19,16 +19,8 @@ using vespalib::makeLambdaTask; namespace proton { -DocumentSubDBCollection::Config::Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed, - size_t fixedAttributeTotalSkew, - CompactionStrategy compaction_strategy, - size_t numSearchThreads) - : _readyGrowth(ready), - _notReadyGrowth(notReady), - _removedGrowth(removed), - _compaction_strategy(compaction_strategy), - _fixedAttributeTotalSkew(fixedAttributeTotalSkew), - _numSearchThreads(numSearchThreads) +DocumentSubDBCollection::Config::Config(size_t numSearchThreads) + : _numSearchThreads(numSearchThreads) { } DocumentSubDBCollection::DocumentSubDBCollection( @@ -69,7 +61,6 @@ DocumentSubDBCollection::DocumentSubDBCollection( SearchableDocSubDB::Config( FastAccessDocSubDB::Config( StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir, - cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _readySubDbId, SubDbType::READY), true, true, false), cfg.getNumSearchThreads()), @@ -79,15 +70,14 @@ DocumentSubDBCollection::DocumentSubDBCollection( _subDBs.push_back (new StoreOnlyDocSubDB( - StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, cfg.getRemovedGrowth(), - cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _remSubDbId, SubDbType::REMOVED), + StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, + _remSubDbId, SubDbType::REMOVED), context)); _subDBs.push_back (new FastAccessDocSubDB( FastAccessDocSubDB::Config( StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir, - cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _notReadySubDbId, SubDbType::NOTREADY), true, true, true), FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService))); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index c64d0f1a6a8..c86370c942b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -5,8 +5,6 @@ #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> -#include <vespa/searchcommon/common/compaction_strategy.h> -#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/util/varholder.h> #include <mutex> @@ -62,24 +60,9 @@ public: using SerialNum = search::SerialNum; class Config { public: - using CompactionStrategy = search::CompactionStrategy; - using GrowStrategy = search::GrowStrategy; - Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed, - size_t fixedAttributeTotalSkew, - CompactionStrategy compaction_strategy, - size_t numSearchThreads); - GrowStrategy getReadyGrowth() const { return _readyGrowth; } - GrowStrategy getNotReadyGrowth() const { return _notReadyGrowth; } - GrowStrategy getRemovedGrowth() const { return _removedGrowth; } - CompactionStrategy get_compaction_strategy() const { return _compaction_strategy; } - size_t getNumSearchThreads() const { return _numSearchThreads; } - size_t getFixedAttributeTotalSkew() const { return _fixedAttributeTotalSkew; } + Config(size_t numSearchThreads); + size_t getNumSearchThreads() const noexcept { return _numSearchThreads; } private: - const GrowStrategy _readyGrowth; - const GrowStrategy _notReadyGrowth; - const GrowStrategy _removedGrowth; - const CompactionStrategy _compaction_strategy; - const size_t _fixedAttributeTotalSkew; const size_t _numSearchThreads; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 5bc6437b4c6..ca2a98d43e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -13,6 +13,7 @@ #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h> @@ -62,6 +63,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co DocumentMetaStore::SP documentMetaStore, std::shared_ptr<AttributeManager::SP> attrMgrResult) const { + AllocStrategy alloc_strategy = configSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); IAttributeFactory::SP attrFactory = std::make_shared<AttributeFactory>(); AttributeManager::SP baseAttrMgr = std::make_shared<AttributeManager>(_baseDir + "/attribute", @@ -77,9 +79,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co documentMetaStore, baseAttrMgr, (_hasAttributes ? configSnapshot.getAttributesConfig() : AttributesConfig()), - _attributeGrow, - _attributeGrowNumDocs, - _attribute_compaction_strategy, + alloc_strategy, _fastAccessAttributesOnly, _writeService.master(), attrMgrResult); @@ -102,11 +102,10 @@ FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult) AttributeCollectionSpec::UP -FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, SerialNum serialNum) const +FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const { uint32_t docIdLimit(_dms->getCommittedDocIdLimit()); - AttributeCollectionSpecFactory factory(_attributeGrow, - _attributeGrowNumDocs, _attribute_compaction_strategy, _fastAccessAttributesOnly); + AttributeCollectionSpecFactory factory(alloc_strategy, _fastAccessAttributesOnly); return factory.create(attrCfg, docIdLimit, serialNum); } @@ -247,7 +246,8 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const { (void) resolver; - reconfigure(newConfigSnapshot.getStoreConfig()); + AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy); IReprocessingTask::List tasks; /* * If attribute manager should change then document retriever @@ -263,7 +263,7 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const std::make_unique<AttributeWriterFactory>(), getSubDbName()); proton::IAttributeManager::SP oldMgr = extractAttributeManager(_fastAccessFeedView.get()); AttributeCollectionSpec::UP attrSpec = - createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum); + createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum); IReprocessingInitializer::UP initializer = configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec); if (initializer->hasReprocessors()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 2bf574aba10..7e2ac3a67cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -84,7 +84,7 @@ protected: MetricsWireService &_metricsWireService; DocIdLimit _docIdLimit; - AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, SerialNum serialNum) const; + AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const; AttributeManager::SP getAndResetInitAttributeManager(); virtual IFlushTargetList getFlushTargetsInternal() override; void reconfigureAttributeMetrics(const IAttributeManager &newMgr, const IAttributeManager &oldMgr); diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 4fc241571ac..bb6c5423175 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -27,7 +27,8 @@ ReconfigParams::configHasChanged() const _res.tuneFileDocumentDBChanged || _res.schemaChanged || _res.maintenanceChanged || - _res.storeChanged; + _res.storeChanged || + _res.alloc_config_changed; } bool @@ -51,7 +52,7 @@ ReconfigParams::shouldIndexManagerChange() const bool ReconfigParams::shouldAttributeManagerChange() const { - return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged; + return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged || _res.alloc_config_changed; } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 2768c7ea337..51e6f8e45df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -7,6 +7,7 @@ #include "i_document_subdb_owner.h" #include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> #include <vespa/searchcore/proton/index/index_manager_initializer.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -143,7 +144,8 @@ IReprocessingTask::List SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) { - StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig()); + AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy); IReprocessingTask::List tasks; applyFlushConfig(newConfigSnapshot.getMaintenanceConfigSP()->getFlushConfig()); if (params.shouldMatchersChange() && _addMetrics) { @@ -152,7 +154,7 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const if (params.shouldAttributeManagerChange()) { proton::IAttributeManager::SP oldMgr = getAttributeManager(); AttributeCollectionSpec::UP attrSpec = - createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum); + createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum); IReprocessingInitializer::UP initializer = _configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec, params, resolver); if (initializer && initializer->hasReprocessors()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 2da0645aef1..9b504c74635 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -10,6 +10,7 @@ #include "storeonlydocsubdb.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/docsummary/summaryflushtarget.h> #include <vespa/searchcore/proton/docsummary/summarymanagerinitializer.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h> @@ -57,15 +58,10 @@ IIndexWriter::SP nullIndexWriter; StoreOnlyDocSubDB::Config::Config(const DocTypeName &docTypeName, const vespalib::string &subName, const vespalib::string &baseDir, - const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, - const search::CompactionStrategy& attribute_compaction_strategy, uint32_t subDbId, SubDbType subDbType) : _docTypeName(docTypeName), _subName(subName), _baseDir(baseDir + "/" + subName), - _attributeGrow(attributeGrow), - _attributeGrowNumDocs(attributeGrowNumDocs), - _attribute_compaction_strategy(attribute_compaction_strategy), _subDbId(subDbId), _subDbType(subDbType) { } @@ -102,9 +98,6 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _bucketDB(ctx._bucketDB), _bucketDBHandlerInitializer(ctx._bucketDBHandlerInitializer), _metaStoreCtx(), - _attributeGrow(cfg._attributeGrow), - _attributeGrowNumDocs(cfg._attributeGrowNumDocs), - _attribute_compaction_strategy(cfg._attribute_compaction_strategy), _flushedDocumentMetaStoreSerialNum(0u), _flushedDocumentStoreSerialNum(0u), _dms(), @@ -226,11 +219,12 @@ StoreOnlyDocSubDB::getNewestFlushedSerial() initializer::InitializerTask::SP StoreOnlyDocSubDB:: createSummaryManagerInitializer(const search::LogDocumentStore::Config & storeCfg, + const AllocStrategy& alloc_strategy, const search::TuneFileSummary &tuneFile, search::IBucketizer::SP bucketizer, std::shared_ptr<SummaryManager::SP> result) const { - GrowStrategy grow = _attributeGrow; + GrowStrategy grow = alloc_strategy.get_grow_strategy(); vespalib::string baseDir(_baseDir + "/summary"); return std::make_shared<SummaryManagerInitializer> (grow, baseDir, getSubDbName(), _docTypeName, _writeService.shared(), @@ -249,12 +243,13 @@ StoreOnlyDocSubDB::setupSummaryManager(SummaryManager::SP summaryManager) InitializerTask::SP StoreOnlyDocSubDB:: -createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile, +createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy, + const search::TuneFileAttributes &tuneFile, std::shared_ptr<DocumentMetaStoreInitializerResult::SP> result) const { - GrowStrategy grow = _attributeGrow; + GrowStrategy grow = alloc_strategy.get_grow_strategy(); // Amortize memory spike cost over N docs - grow.setDocsGrowDelta(grow.getDocsGrowDelta() + _attributeGrowNumDocs); + grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); vespalib::string baseDir(_baseDir + "/documentmetastore"); vespalib::string name = DocumentMetaStore::getFixedName(); vespalib::string attrFileName = baseDir + "/" + name; // XXX: Wrong @@ -296,10 +291,13 @@ StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot, Ser { auto result = std::make_unique<DocumentSubDbInitializer>(const_cast<StoreOnlyDocSubDB &>(*this), _writeService.master()); - auto dmsInitTask = createDocumentMetaStoreInitializer(configSnapshot.getTuneFileDocumentDBSP()->_attr, + AllocStrategy alloc_strategy = configSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + auto dmsInitTask = createDocumentMetaStoreInitializer(alloc_strategy, + configSnapshot.getTuneFileDocumentDBSP()->_attr, result->writableResult().writableDocumentMetaStore()); result->addDocumentMetaStoreInitTask(dmsInitTask); auto summaryTask = createSummaryManagerInitializer(configSnapshot.getStoreConfig(), + alloc_strategy, configSnapshot.getTuneFileDocumentDBSP()->_summary, result->result().documentMetaStore()->documentMetaStore(), result->writableResult().writableSummaryManager()); @@ -413,14 +411,22 @@ StoreOnlyDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const (void) params; (void) resolver; assert(_writeService.master().isCurrentThread()); - reconfigure(newConfigSnapshot.getStoreConfig()); + AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy); initFeedView(newConfigSnapshot); return IReprocessingTask::List(); } void -StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config) +StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, const AllocStrategy& alloc_strategy) { + auto cfg = _dms->getConfig(); + GrowStrategy grow = alloc_strategy.get_grow_strategy(); + // Amortize memory spike cost over N docs + grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); + cfg.setGrowStrategy(grow); + cfg.setCompactionStrategy(alloc_strategy.get_compaction_strategy()); + _dms->update_config(cfg); // Update grow and compaction config _rSummaryMgr->reconfigure(config); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index a0b278eb491..37229bd551c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -20,6 +20,7 @@ namespace proton { +class AllocStrategy; struct DocumentDBTaggedMetrics; class DocumentMetaStoreInitializerResult; class FeedHandler; @@ -85,16 +86,11 @@ public: const DocTypeName _docTypeName; const vespalib::string _subName; const vespalib::string _baseDir; - const search::GrowStrategy _attributeGrow; - const size_t _attributeGrowNumDocs; - const search::CompactionStrategy _attribute_compaction_strategy; const uint32_t _subDbId; const SubDbType _subDbType; Config(const DocTypeName &docTypeName, const vespalib::string &subName, const vespalib::string &baseDir, - const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, - const search::CompactionStrategy& attribute_compaction_strategy, uint32_t subDbId, SubDbType subDbType); ~Config(); }; @@ -133,9 +129,6 @@ protected: BucketDBOwner::SP _bucketDB; bucketdb::IBucketDBHandlerInitializer &_bucketDBHandlerInitializer; IDocumentMetaStoreContext::SP _metaStoreCtx; - const search::GrowStrategy _attributeGrow; - const size_t _attributeGrowNumDocs; - const search::CompactionStrategy _attribute_compaction_strategy; // The following two serial numbers reflect state at program startup // and are used by replay logic. SerialNum _flushedDocumentMetaStoreSerialNum; @@ -168,6 +161,7 @@ protected: std::shared_ptr<initializer::InitializerTask> createSummaryManagerInitializer(const search::LogDocumentStore::Config & protonSummaryCfg, + const AllocStrategy& alloc_strategy, const search::TuneFileSummary &tuneFile, search::IBucketizer::SP bucketizer, std::shared_ptr<SummaryManager::SP> result) const; @@ -175,7 +169,8 @@ protected: void setupSummaryManager(SummaryManager::SP summaryManager); std::shared_ptr<initializer::InitializerTask> - createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile, + createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy, + const search::TuneFileAttributes &tuneFile, std::shared_ptr<std::shared_ptr<DocumentMetaStoreInitializerResult>> result) const; void setupDocumentMetaStore(std::shared_ptr<DocumentMetaStoreInitializerResult> dmsResult); @@ -185,7 +180,8 @@ protected: StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams(); vespalib::string getSubDbName() const; - void reconfigure(const search::LogDocumentStore::Config & protonConfig); + void reconfigure(const search::LogDocumentStore::Config & protonConfig, + const AllocStrategy& alloc_strategy); public: StoreOnlyDocSubDB(const Config &cfg, const Context &ctx); ~StoreOnlyDocSubDB() override; diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 335c7aca24c..17cb91ac4ce 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -10,6 +10,7 @@ #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/config-imported-fields.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/server/threading_service_config.h> using document::DocumenttypesConfig; @@ -47,6 +48,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, _maintenance(std::make_shared<DocumentDBMaintenanceConfig>()), _store(), _threading_service_config(std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1))), + _alloc_config(std::make_shared<const AllocConfig>()), _configId(configId), _docTypeName(docTypeName) { } @@ -70,6 +72,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) _maintenance(cfg.getMaintenanceConfigSP()), _store(cfg.getStoreConfig()), _threading_service_config(cfg.get_threading_service_config_shared_ptr()), + _alloc_config(cfg.get_alloc_config_shared_ptr()), _configId(cfg.getConfigId()), _docTypeName(cfg.getDocTypeName()) {} @@ -97,6 +100,7 @@ DocumentDBConfigBuilder::build() _maintenance, _store, _threading_service_config, + _alloc_config, _configId, _docTypeName); } diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 218a7c56fa9..706e14e73db 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -28,6 +28,7 @@ private: DocumentDBConfig::MaintenanceConfigSP _maintenance; search::LogDocumentStore::Config _store; std::shared_ptr<const ThreadingServiceConfig> _threading_service_config; + std::shared_ptr<const AllocConfig> _alloc_config; vespalib::string _configId; vespalib::string _docTypeName; diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index dde6b14121d..36a697eaa12 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -226,4 +226,24 @@ TEST_F("Compaction is not executed when free lists are used", EXPECT_EQUAL(1001u, afterSpace.dead()); } +TEST_F("Compaction is peformed when compaction strategy is changed to enable compaction", + Fixture(compactAddressSpaceAttributeConfig(false))) +{ + populate_and_hammer(f, true); + AddressSpace after1 = f.getMultiValueAddressSpaceUsage("after1"); + // 100 * 1000 dead arrays due to new values for docids + // 1 reserved array accounted as dead + EXPECT_EQUAL(100001u, after1.dead()); + f._v->update_config(compactAddressSpaceAttributeConfig(true)); + auto old_dead = after1.dead(); + AddressSpace after2 = f.getMultiValueAddressSpaceUsage("after2"); + while (after2.dead() < old_dead) { + old_dead = after2.dead(); + f._v->commit(); // new commit might trigger further compaction + after2 = f.getMultiValueAddressSpaceUsage("after2"); + } + // DEAD_ARRAYS_SLACK in multi value mapping is is 64k + EXPECT_GREATER(65536u, after2.dead()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 2168bbe4276..d2574bd32a2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -21,6 +21,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/util/logutil.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attributevector"); @@ -769,6 +770,31 @@ AttributeVector::logEnumStoreEvent(const char *reason, const char *stage) EV_STATE(eventName.c_str(), jstr.toString().data()); } +void +AttributeVector::drain_hold(uint64_t hold_limit) +{ + incGeneration(); + for (int retry = 0; ; ++retry) { + removeAllOldGenerations(); + updateStat(true); + if (_status.getOnHold() <= hold_limit) { + return; + } + std::this_thread::sleep_for(retry < 20 ? 20ms : 100ms); + } +} + +void +AttributeVector::update_config(const Config& cfg) +{ + commit(true); + drain_hold(1024 * 1024); // Wait until 1MiB or less on hold + _config.setGrowStrategy(cfg.getGrowStrategy()); + _config.setCompactionStrategy(cfg.getCompactionStrategy()); + commit(); // might trigger compaction if compaction strategy changed + drain_hold(1024 * 1024); // Wait until 1MiB or less on hold +} + template bool AttributeVector::append<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t, bool); template bool AttributeVector::update<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &); template bool AttributeVector::remove<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 4fc6589850e..f308ee8d024 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -393,6 +393,7 @@ public: /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } const Config &getConfig() const { return _config; } + void update_config(const Config& cfg); BasicType getInternalBasicType() const { return _config.basicType(); } CollectionType getInternalCollectionType() const { return _config.collectionType(); } const BaseName & getBaseFileName() const { return _baseFileName; } @@ -667,6 +668,8 @@ public: static bool isEnumerated(const vespalib::GenericHeader &header); virtual vespalib::MemoryUsage getChangeVectorMemoryUsage() const; + + void drain_hold(uint64_t hold_limit); }; } |