summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2021-02-02 17:13:58 +0100
committerGitHub <noreply@github.com>2021-02-02 17:13:58 +0100
commitd1daad830a576f9147b709523670635420fd49d3 (patch)
tree0a542eb77f3ba9eb8d3c1af665afc14cf2c67021
parent941bf25aea1ff089f54b62ec78af5d4f14d2af35 (diff)
parent8bf59ff05217b43096550ad3a0eb46de6d113c12 (diff)
Merge pull request #16343 from vespa-engine/toregge/improve-handling-of-changed-alloc-config
Improve handling of changed alloc config.
-rw-r--r--searchcommon/src/vespa/searchcommon/common/CMakeLists.txt2
-rw-r--r--searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp15
-rw-r--r--searchcommon/src/vespa/searchcommon/common/compaction_strategy.h3
-rw-r--r--searchcommon/src/vespa/searchcommon/common/growstrategy.cpp18
-rw-r--r--searchcommon/src/vespa/searchcommon/common/growstrategy.h3
-rw-r--r--searchcore/CMakeLists.txt1
-rw-r--r--searchcore/src/apps/tests/persistenceconformance_test.cpp2
-rw-r--r--searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp2
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp10
-rw-r--r--searchcore/src/tests/proton/attribute/attribute_test.cpp2
-rw-r--r--searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt9
-rw-r--r--searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp35
-rw-r--r--searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp7
-rw-r--r--searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp3
-rw-r--r--searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp2
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp14
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h11
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp10
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h11
-rw-r--r--searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt2
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp56
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/alloc_config.h34
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp41
-rw-r--r--searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h41
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdb.cpp42
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp17
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h7
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp21
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp18
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h21
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp16
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h2
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp5
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp6
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp36
-rw-r--r--searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h16
-rw-r--r--searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp4
-rw-r--r--searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h1
-rw-r--r--searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp20
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attributevector.cpp26
-rw-r--r--searchlib/src/vespa/searchlib/attribute/attributevector.h3
42 files changed, 442 insertions, 157 deletions
diff --git a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt
index 23e6e8dd394..bf5686280e6 100644
--- a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt
+++ b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt
@@ -1,7 +1,9 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(searchcommon_searchcommon_common OBJECT
SOURCES
+ compaction_strategy.cpp
datatype.cpp
+ growstrategy.cpp
schema.cpp
schemaconfigurer.cpp
DEPENDS
diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp
new file mode 100644
index 00000000000..c3377ed5857
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp
@@ -0,0 +1,15 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "compaction_strategy.h"
+#include <iostream>
+namespace search {
+
+std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy)
+{
+ os << "{maxDeadBytesRatio=" << compaction_strategy.getMaxDeadBytesRatio() <<
+ ", maxDeadAddressSpaceRatio=" << compaction_strategy.getMaxDeadAddressSpaceRatio() <<
+ "}";
+ return os;
+}
+
+}
diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h
index 0b3caf44481..ae354a4c4d2 100644
--- a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h
+++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h
@@ -3,6 +3,7 @@
#pragma once
#include <stdint.h>
+#include <iosfwd>
namespace search {
@@ -34,4 +35,6 @@ public:
bool operator!=(const CompactionStrategy & rhs) const { return !(operator==(rhs)); }
};
+std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy);
+
} // namespace search
diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp b/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp
new file mode 100644
index 00000000000..534be3060a7
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp
@@ -0,0 +1,18 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "growstrategy.h"
+#include <iostream>
+
+namespace search {
+
+std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy)
+{
+ os << "{docsInitialCapacity=" << grow_strategy.getDocsInitialCapacity() <<
+ ", docsGrowFactor=" << grow_strategy.getDocsGrowFactor() <<
+ ", docsGrowDelta=" << grow_strategy.getDocsGrowDelta() <<
+ ", multiValueAllocGrowFactor=" << grow_strategy.getMultiValueAllocGrowFactor() <<
+ "}";
+ return os;
+}
+
+}
diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.h b/searchcommon/src/vespa/searchcommon/common/growstrategy.h
index 13220741e51..f7c5c030d95 100644
--- a/searchcommon/src/vespa/searchcommon/common/growstrategy.h
+++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.h
@@ -4,6 +4,7 @@
#include <vespa/vespalib/util/growstrategy.h>
#include <cstdint>
+#include <iosfwd>
namespace search {
@@ -54,5 +55,7 @@ public:
}
};
+std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy);
+
}
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt
index 04cface5d40..d77bc69ba62 100644
--- a/searchcore/CMakeLists.txt
+++ b/searchcore/CMakeLists.txt
@@ -69,6 +69,7 @@ vespa_define_module(
src/tests/proton/attribute/imported_attributes_repo
src/tests/proton/bucketdb/bucketdb
src/tests/proton/common
+ src/tests/proton/common/alloc_config
src/tests/proton/common/attribute_updater
src/tests/proton/common/document_type_inspector
src/tests/proton/common/hw_info_sampler
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
index 4715ff80d03..b47a1954c6f 100644
--- a/searchcore/src/apps/tests/persistenceconformance_test.cpp
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -13,6 +13,7 @@
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/document/test/make_bucket_space.h>
#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/matching/querylimiter.h>
#include <vespa/searchcore/proton/metrics/metricswireservice.h>
@@ -145,6 +146,7 @@ public:
std::make_shared<DocumentDBMaintenanceConfig>(),
search::LogDocumentStore::Config(),
std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)),
+ std::make_shared<const AllocConfig>(),
"client",
docTypeName.getName());
}
diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp
index 90942e9aef4..6cca2e4bd48 100644
--- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp
+++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp
@@ -38,6 +38,7 @@
#include <vespa/messagebus/testlib/slobrok.h>
#include <vespa/metrics/config-metricsmanager.h>
#include <vespa/searchcommon/common/schemaconfigurer.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/matching/querylimiter.h>
#include <vespa/searchcore/proton/metrics/metricswireservice.h>
@@ -198,6 +199,7 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume
std::make_shared<DocumentDBMaintenanceConfig>(),
search::LogDocumentStore::Config(),
std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)),
+ std::make_shared<const AllocConfig>(),
"client",
doc_type_name.getName());
}
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index d07c25c2a9e..3987a8685ea 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -232,9 +232,7 @@ struct ParallelAttributeManager
InitializerTask::SP documentMetaStoreInitTask;
BucketDBOwner::SP bucketDbOwner;
DocumentMetaStore::SP documentMetaStore;
- search::GrowStrategy attributeGrow;
- size_t attributeGrowNumDocs;
- search::CompactionStrategy attribute_compaction_strategy;
+ AllocStrategy alloc_strategy;
bool fastAccessAttributesOnly;
std::shared_ptr<AttributeManager::SP> mgr;
vespalib::ThreadStackExecutor masterExecutor;
@@ -251,16 +249,14 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria
: documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()),
bucketDbOwner(std::make_shared<BucketDBOwner>()),
documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)),
- attributeGrow(),
- attributeGrowNumDocs(1),
- attribute_compaction_strategy(),
+ alloc_strategy(),
fastAccessAttributesOnly(false),
mgr(std::make_shared<AttributeManager::SP>()),
masterExecutor(1, 128 * 1024),
master(masterExecutor),
initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask,
documentMetaStore, baseAttrMgr, attrCfg,
- attributeGrow, attributeGrowNumDocs, attribute_compaction_strategy,
+ alloc_strategy,
fastAccessAttributesOnly, master, mgr))
{
documentMetaStore->setCommittedDocIdLimit(docIdLimit);
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index a9829f3e5f7..ebd3e27aeca 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -535,7 +535,7 @@ public:
AttributeCollectionSpecFactory _factory;
AttributeCollectionSpecTest(bool fastAccessOnly)
: _builder(),
- _factory(search::GrowStrategy(), 100, search::CompactionStrategy(), fastAccessOnly)
+ _factory(AllocStrategy(search::GrowStrategy(), search::CompactionStrategy(), 100), fastAccessOnly)
{
addAttribute("a1", false);
addAttribute("a2", true);
diff --git a/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt b/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt
new file mode 100644
index 00000000000..26a2dc72cc2
--- /dev/null
+++ b/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_alloc_config_test_app TEST
+ SOURCES
+ alloc_config_test.cpp
+ DEPENDS
+ searchcore_pcommon
+ GTest::GTest
+)
+vespa_add_test(NAME searchcore_alloc_config_test_app COMMAND searchcore_alloc_config_test_app)
diff --git a/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp
new file mode 100644
index 00000000000..18a0ee47a47
--- /dev/null
+++ b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp
@@ -0,0 +1,35 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcore/proton/common/alloc_config.h>
+#include <vespa/searchcore/proton/common/subdbtype.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using proton::AllocConfig;
+using proton::AllocStrategy;
+using proton::SubDbType;
+using search::CompactionStrategy;
+using search::GrowStrategy;
+
+namespace {
+
+CompactionStrategy baseline_compaction_strategy(0.2, 0.25);
+
+GrowStrategy make_grow_strategy(uint32_t initial_docs) {
+ return GrowStrategy(initial_docs, 0.1, 1, 0.15);
+}
+
+AllocStrategy make_alloc_strategy(uint32_t initial_docs) {
+ return AllocStrategy(make_grow_strategy(initial_docs), baseline_compaction_strategy, 10000);
+}
+
+};
+
+TEST(AllocConfigTest, can_make_allocation_strategy_for_sub_dbs)
+{
+ AllocConfig config(make_alloc_strategy(10000000), 5, 2);
+ EXPECT_EQ(make_alloc_strategy(20000000), config.make_alloc_strategy(SubDbType::READY));
+ EXPECT_EQ(make_alloc_strategy(100000), config.make_alloc_strategy(SubDbType::REMOVED));
+ EXPECT_EQ(make_alloc_strategy(30000000), config.make_alloc_strategy(SubDbType::NOTREADY));
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index d231040aeda..8f6b2f0bc10 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -613,6 +613,12 @@ TEST("require that attribute manager (imported attributes) should change when vi
EXPECT_TRUE(params.shouldAttributeManagerChange());
}
+TEST("require that attribute manager should change when alloc config has changed")
+{
+ ReconfigParams params(CCR().set_alloc_config_changed(true));
+ EXPECT_TRUE(params.shouldAttributeManagerChange());
+}
+
void
assertMaintenanceControllerShouldNotChange(DocumentDBConfig::ComparisonResult result)
{
@@ -684,6 +690,7 @@ TEST("require that subdbs should change if relevant config changed")
TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true)));
TEST_DO(assertSubDbsShouldChange(CCR().setOnnxModelsChanged(true)));
TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true)));
+ TEST_DO(assertSubDbsShouldChange(CCR().set_alloc_config_changed(true)));
}
TEST_MAIN()
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 64178303b40..40a9656ae41 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -123,8 +123,7 @@ struct MyStoreOnlyConfig
: _cfg(DocTypeName(DOCTYPE_NAME),
SUB_NAME,
BASE_DIR,
- search::GrowStrategy(),
- 0, search::CompactionStrategy(), 0, SubDbType::READY)
+ 0, SubDbType::READY)
{
}
};
diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index 5a0cfc18c78..4f8e8e8aa8c 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -9,6 +9,7 @@
#include <vespa/config-summarymap.h>
#include <vespa/document/repo/documenttyperepo.h>
#include <vespa/fileacquirer/config-filedistributorrpc.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/server/bootstrapconfig.h>
#include <vespa/searchcore/proton/server/bootstrapconfigmanager.h>
#include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
@@ -103,6 +104,7 @@ struct DBConfigFixture {
std::make_shared<DocumentDBMaintenanceConfig>(),
search::LogDocumentStore::Config(),
std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)),
+ std::make_shared<const AllocConfig>(),
configId,
docTypeName);
}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
index 96db6205bac..4189688ea81 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp
@@ -9,13 +9,9 @@ using search::GrowStrategy;
namespace proton {
AttributeCollectionSpecFactory::AttributeCollectionSpecFactory(
- const search::GrowStrategy &growStrategy,
- size_t growNumDocs,
- const search::CompactionStrategy& compaction_strategy,
+ const AllocStrategy &alloc_strategy,
bool fastAccessOnly)
- : _growStrategy(growStrategy),
- _growNumDocs(growNumDocs),
- _compaction_strategy(compaction_strategy),
+ : _alloc_strategy(alloc_strategy),
_fastAccessOnly(fastAccessOnly)
{
}
@@ -27,8 +23,8 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg,
{
AttributeCollectionSpec::AttributeList attrs;
// Amortize memory spike cost over N docs
- const size_t skew = _growNumDocs/(attrCfg.attribute.size()+1);
- GrowStrategy grow = _growStrategy;
+ const size_t skew = _alloc_strategy.get_amortize_count()/(attrCfg.attribute.size()+1);
+ GrowStrategy grow = _alloc_strategy.get_grow_strategy();
grow.setDocsInitialCapacity(std::max(grow.getDocsInitialCapacity(),docIdLimit));
for (const auto &attr : attrCfg.attribute) {
search::attribute::Config cfg = ConfigConverter::convert(attr);
@@ -37,7 +33,7 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg,
}
grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew);
cfg.setGrowStrategy(grow);
- cfg.setCompactionStrategy(_compaction_strategy);
+ cfg.setCompactionStrategy(_alloc_strategy.get_compaction_strategy());
attrs.push_back(AttributeSpec(attr.name, cfg));
}
return std::make_unique<AttributeCollectionSpec>(attrs, docIdLimit, serialNum);
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
index 9e877bedf28..074c56448f3 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h
@@ -4,8 +4,7 @@
#include "attribute_collection_spec.h"
#include <vespa/searchcommon/attribute/config.h>
-#include <vespa/searchcommon/common/compaction_strategy.h>
-#include <vespa/searchcommon/common/growstrategy.h>
+#include <vespa/searchcore/proton/common/alloc_strategy.h>
#include <vespa/searchlib/common/serialnum.h>
#include <vespa/config-attributes.h>
@@ -20,15 +19,11 @@ class AttributeCollectionSpecFactory
private:
typedef vespa::config::search::AttributesConfig AttributesConfig;
- const search::GrowStrategy _growStrategy;
- const size_t _growNumDocs;
- const search::CompactionStrategy _compaction_strategy;
+ const AllocStrategy _alloc_strategy;
const bool _fastAccessOnly;
public:
- AttributeCollectionSpecFactory(const search::GrowStrategy &growStrategy,
- size_t growNumDocs,
- const search::CompactionStrategy& compaction_strategy,
+ AttributeCollectionSpecFactory(const AllocStrategy& alloc_strategy,
bool fastAccessOnly);
AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg,
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
index cddf8a2098d..109a4bb2192 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp
@@ -136,7 +136,7 @@ AttributeCollectionSpec::UP
AttributeManagerInitializer::createAttributeSpec() const
{
uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them
- AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _compaction_strategy, _fastAccessAttributesOnly);
+ AttributeCollectionSpecFactory factory(_alloc_strategy, _fastAccessAttributesOnly);
return factory.create(_attrCfg, docIdLimit, _configSerialNum);
}
@@ -145,9 +145,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN
DocumentMetaStore::SP documentMetaStore,
AttributeManager::SP baseAttrMgr,
const AttributesConfig &attrCfg,
- const GrowStrategy &attributeGrow,
- size_t attributeGrowNumDocs,
- const CompactionStrategy& compaction_strategy,
+ const AllocStrategy& alloc_strategy,
bool fastAccessAttributesOnly,
searchcorespi::index::IThreadService &master,
std::shared_ptr<AttributeManager::SP> attrMgrResult)
@@ -155,9 +153,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN
_documentMetaStore(documentMetaStore),
_attrMgr(),
_attrCfg(attrCfg),
- _attributeGrow(attributeGrow),
- _attributeGrowNumDocs(attributeGrowNumDocs),
- _compaction_strategy(compaction_strategy),
+ _alloc_strategy(alloc_strategy),
_fastAccessAttributesOnly(fastAccessAttributesOnly),
_master(master),
_attributesResult(),
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
index 7c8c34a0d4d..f74a0f1519d 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h
@@ -4,10 +4,9 @@
#include "attributemanager.h"
#include "initialized_attributes_result.h"
-#include <vespa/searchcommon/common/compaction_strategy.h>
-#include <vespa/searchcommon/common/growstrategy.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastore.h>
#include <vespa/searchcore/proton/initializer/initializer_task.h>
+#include <vespa/searchcore/proton/common/alloc_strategy.h>
#include <vespa/searchlib/common/serialnum.h>
#include <vespa/config-attributes.h>
@@ -25,9 +24,7 @@ private:
DocumentMetaStore::SP _documentMetaStore;
AttributeManager::SP _attrMgr;
vespa::config::search::AttributesConfig _attrCfg;
- search::GrowStrategy _attributeGrow;
- size_t _attributeGrowNumDocs;
- search::CompactionStrategy _compaction_strategy;
+ AllocStrategy _alloc_strategy;
bool _fastAccessAttributesOnly;
searchcorespi::index::IThreadService &_master;
InitializedAttributesResult _attributesResult;
@@ -41,9 +38,7 @@ public:
DocumentMetaStore::SP documentMetaStore,
AttributeManager::SP baseAttrMgr,
const vespa::config::search::AttributesConfig &attrCfg,
- const search::GrowStrategy &attributeGrow,
- size_t attributeGrowNumDocs,
- const search::CompactionStrategy& compaction_strategy,
+ const AllocStrategy& alloc_strategy,
bool fastAccessAttributesOnly,
searchcorespi::index::IThreadService &master,
std::shared_ptr<AttributeManager::SP> attrMgrResult);
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index 0b00e274522..f635ee34a04 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -177,10 +177,14 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr,
auto shrinker = wrap->getShrinker();
assert(shrinker);
addAttribute(AttributeWrap::normalAttribute(av), shrinker);
+ auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix());
+ auto cfg = aspec.getConfig();
+ _attributeFieldWriter.execute(id, [av, cfg]() { av->update_config(cfg); });
} else {
toBeAdded.push_back(aspec);
}
}
+ _attributeFieldWriter.sync();
}
void
diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
index d59a4075dce..a91c35f0485 100644
--- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
@@ -1,6 +1,8 @@
# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(searchcore_pcommon STATIC
SOURCES
+ alloc_config.cpp
+ alloc_strategy.cpp
attribute_updater.cpp
attributefieldvaluenode.cpp
cachedselect.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp
new file mode 100644
index 00000000000..1611d00fb0f
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp
@@ -0,0 +1,56 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "alloc_config.h"
+#include <vespa/searchcore/proton/common/subdbtype.h>
+#include <algorithm>
+
+using search::CompactionStrategy;
+using search::GrowStrategy;
+
+namespace proton {
+
+AllocConfig::AllocConfig(const AllocStrategy& alloc_strategy,
+ uint32_t redundancy, uint32_t searchable_copies)
+ : _alloc_strategy(alloc_strategy),
+ _redundancy(redundancy),
+ _searchable_copies(searchable_copies)
+{
+}
+
+AllocConfig::AllocConfig()
+ : AllocConfig(AllocStrategy(), 1, 1)
+{
+}
+
+AllocConfig::~AllocConfig() = default;
+
+bool
+AllocConfig::operator==(const AllocConfig &rhs) const noexcept
+{
+ return ((_alloc_strategy == rhs._alloc_strategy) &&
+ (_redundancy == rhs._redundancy) &&
+ (_searchable_copies == rhs._searchable_copies));
+}
+
+AllocStrategy
+AllocConfig::make_alloc_strategy(SubDbType sub_db_type) const
+{
+ auto &baseline_grow_strategy = _alloc_strategy.get_grow_strategy();
+ size_t initial_capacity = baseline_grow_strategy.getDocsInitialCapacity();
+ switch (sub_db_type) {
+ case SubDbType::READY:
+ initial_capacity *= _searchable_copies;
+ break;
+ case SubDbType::NOTREADY:
+ initial_capacity *= (_redundancy - _searchable_copies);
+ break;
+ case SubDbType::REMOVED:
+ default:
+ initial_capacity = std::max(1024ul, initial_capacity / 100);
+ break;
+ }
+ GrowStrategy grow_strategy(initial_capacity, baseline_grow_strategy.getDocsGrowFactor(), baseline_grow_strategy.getDocsGrowDelta(), baseline_grow_strategy.getMultiValueAllocGrowFactor());
+ return AllocStrategy(grow_strategy, _alloc_strategy.get_compaction_strategy(), _alloc_strategy.get_amortize_count());
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.h b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h
new file mode 100644
index 00000000000..25b953b9871
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h
@@ -0,0 +1,34 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "alloc_strategy.h"
+
+namespace proton {
+
+enum class SubDbType;
+
+/*
+ * Class representing allocation config for proton which can be used
+ * to make an allocation strategy for large data structures owned by a
+ * document sub db (e.g. attribute vectors, document meta store).
+ */
+class AllocConfig
+{
+ AllocStrategy _alloc_strategy; // baseline before adjusting for redundancy / searchable copies
+ const uint32_t _redundancy;
+ const uint32_t _searchable_copies;
+
+public:
+ AllocConfig(const AllocStrategy& alloc_strategy, uint32_t redundancy, uint32_t searchable_copies);
+ AllocConfig();
+ ~AllocConfig();
+
+ bool operator==(const AllocConfig &rhs) const noexcept;
+ bool operator!=(const AllocConfig &rhs) const noexcept {
+ return !operator==(rhs);
+ }
+ AllocStrategy make_alloc_strategy(SubDbType sub_db_type) const;
+};
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp
new file mode 100644
index 00000000000..3af72757ccb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp
@@ -0,0 +1,41 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "alloc_strategy.h"
+#include <iostream>
+
+using search::CompactionStrategy;
+using search::GrowStrategy;
+
+namespace proton {
+
+AllocStrategy::AllocStrategy(const GrowStrategy& grow_strategy,
+ const CompactionStrategy& compaction_strategy,
+ uint32_t amortize_count)
+ : _grow_strategy(grow_strategy),
+ _compaction_strategy(compaction_strategy),
+ _amortize_count(amortize_count)
+{
+}
+
+AllocStrategy::AllocStrategy()
+ : AllocStrategy(GrowStrategy(), CompactionStrategy(), 10000)
+{
+}
+
+AllocStrategy::~AllocStrategy() = default;
+
+bool
+AllocStrategy::operator==(const AllocStrategy &rhs) const noexcept
+{
+ return ((_grow_strategy == rhs._grow_strategy) &&
+ (_compaction_strategy == rhs._compaction_strategy) &&
+ (_amortize_count == rhs._amortize_count));
+}
+
+std::ostream& operator<<(std::ostream& os, const AllocStrategy&alloc_strategy)
+{
+ os << "{ grow_strategy=" << alloc_strategy.get_grow_strategy() << ", compaction_strategy=" << alloc_strategy.get_compaction_strategy() << ", amortize_count=" << alloc_strategy.get_amortize_count() << "}";
+ return os;
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h
new file mode 100644
index 00000000000..74bcc1772ee
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h
@@ -0,0 +1,41 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/common/compaction_strategy.h>
+#include <vespa/searchcommon/common/growstrategy.h>
+#include <iosfwd>
+
+namespace proton {
+
+/*
+ * Class representing allocation strategy for large data structures
+ * owned by a document sub db (e.g. attribute vectors, document meta store).
+ */
+class AllocStrategy
+{
+protected:
+ const search::GrowStrategy _grow_strategy;
+ const search::CompactionStrategy _compaction_strategy;
+ const uint32_t _amortize_count;
+
+public:
+ AllocStrategy(const search::GrowStrategy& grow_strategy,
+ const search::CompactionStrategy& compaction_strategy,
+ uint32_t amortize_count);
+
+ AllocStrategy();
+ ~AllocStrategy();
+
+ bool operator==(const AllocStrategy &rhs) const noexcept;
+ bool operator!=(const AllocStrategy &rhs) const noexcept {
+ return !operator==(rhs);
+ }
+ const search::GrowStrategy& get_grow_strategy() const noexcept { return _grow_strategy; }
+ const search::CompactionStrategy& get_compaction_strategy() const noexcept { return _compaction_strategy; }
+ uint32_t get_amortize_count() const noexcept { return _amortize_count; }
+};
+
+std::ostream& operator<<(std::ostream& os, const AllocStrategy&alloc_strategy);
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 0c5b4ef0d33..166d1e928bb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -70,49 +70,11 @@ namespace proton {
namespace {
constexpr uint32_t indexing_thread_stack_size = 128 * 1024;
-using Allocation = ProtonConfig::Documentdb::Allocation;
-GrowStrategy
-makeGrowStrategy(uint32_t docsInitialCapacity, const Allocation &allocCfg)
-{
- return GrowStrategy(docsInitialCapacity, allocCfg.growfactor, allocCfg.growbias, allocCfg.multivaluegrowfactor);
-}
-
-
-CompactionStrategy
-make_compaction_strategy(const Allocation& alloc_cfg)
-{
- double max_dead_bytes_ratio = alloc_cfg.maxDeadBytesRatio;
- double max_dead_address_space_ratio = alloc_cfg.maxDeadAddressSpaceRatio;
- return CompactionStrategy(max_dead_bytes_ratio, max_dead_address_space_ratio);
-}
-
-DocumentSubDBCollection::Config
-makeSubDBConfig(const ProtonConfig::Distribution & distCfg, const Allocation & allocCfg, size_t numSearcherThreads) {
- size_t initialNumDocs(allocCfg.initialnumdocs);
- GrowStrategy searchableGrowth = makeGrowStrategy(initialNumDocs * distCfg.searchablecopies, allocCfg);
- GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024ul, initialNumDocs/100), allocCfg);
- GrowStrategy notReadyGrowth = makeGrowStrategy(initialNumDocs * (distCfg.redundancy - distCfg.searchablecopies), allocCfg);
- CompactionStrategy compaction_strategy = make_compaction_strategy(allocCfg);
- return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, compaction_strategy, numSearcherThreads);
-}
-
index::IndexConfig
makeIndexConfig(const ProtonConfig::Index & cfg) {
return index::IndexConfig(WarmupConfig(vespalib::from_s(cfg.warmup.time), cfg.warmup.unpack), cfg.maxflushed, cfg.cache.size);
}
-ProtonConfig::Documentdb _G_defaultProtonDocumentDBConfig;
-
-const ProtonConfig::Documentdb *
-findDocumentDB(const ProtonConfig::DocumentdbVector & documentDBs, const vespalib::string & docType) {
- for (const auto & dbCfg : documentDBs) {
- if (dbCfg.inputdoctypename == docType) {
- return & dbCfg;
- }
- }
- return &_G_defaultProtonDocumentDBConfig;
-}
-
class MetricsUpdateHook : public metrics::UpdateHook {
DocumentDB &_db;
public:
@@ -193,9 +155,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
_feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)),
_subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext,
metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir,
- makeSubDBConfig(protonCfg.distribution,
- findDocumentDB(protonCfg.documentdb, docTypeName.getName())->allocation,
- protonCfg.numsearcherthreads),
+ DocumentSubDBCollection::Config(protonCfg.numsearcherthreads),
hwInfo),
_maintenanceController(_writeService.master(), sharedExecutor, _docTypeName),
_lidSpaceCompactionHandlers(),
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
index baa6c8eb450..7c487043b5b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
@@ -14,6 +14,7 @@
#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/searchcore/config/config-onnx-models.h>
#include <vespa/searchcore/proton/attribute/attribute_aspect_delayer.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/common/document_type_inspector.h>
#include <vespa/searchcore/proton/common/indexschema_inspector.h>
@@ -49,7 +50,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult()
storeChanged(false),
visibilityDelayChanged(false),
flushChanged(false),
- threading_service_config_changed(false)
+ threading_service_config_changed(false),
+ alloc_config_changed(false)
{ }
DocumentDBConfig::DocumentDBConfig(
@@ -70,6 +72,7 @@ DocumentDBConfig::DocumentDBConfig(
const DocumentDBMaintenanceConfig::SP &maintenance,
const search::LogDocumentStore::Config & storeConfig,
std::shared_ptr<const ThreadingServiceConfig> threading_service_config,
+ std::shared_ptr<const AllocConfig> alloc_config,
const vespalib::string &configId,
const vespalib::string &docTypeName) noexcept
: _configId(configId),
@@ -91,6 +94,7 @@ DocumentDBConfig::DocumentDBConfig(
_maintenance(maintenance),
_storeConfig(storeConfig),
_threading_service_config(std::move(threading_service_config)),
+ _alloc_config(std::move(alloc_config)),
_orig(),
_delayedAttributeAspects(false)
{ }
@@ -117,6 +121,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg)
_maintenance(cfg._maintenance),
_storeConfig(cfg._storeConfig),
_threading_service_config(cfg._threading_service_config),
+ _alloc_config(cfg._alloc_config),
_orig(cfg._orig),
_delayedAttributeAspects(false)
{ }
@@ -141,7 +146,8 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const
equals<Schema>(_schema.get(), rhs._schema.get()) &&
equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()) &&
_storeConfig == rhs._storeConfig &&
- equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get());
+ equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()) &&
+ equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get());
}
@@ -167,6 +173,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const
retval.visibilityDelayChanged = (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay());
retval.flushChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get(), [](const auto &l, const auto &r) { return l.getFlushConfig() == r.getFlushConfig(); });
retval.threading_service_config_changed = !equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get());
+ retval.alloc_config_changed = !equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get());
return retval;
}
@@ -188,7 +195,8 @@ DocumentDBConfig::valid() const
_tuneFileDocumentDB &&
_schema &&
_maintenance &&
- _threading_service_config;
+ _threading_service_config &&
+ _alloc_config;
}
namespace
@@ -232,6 +240,7 @@ DocumentDBConfig::makeReplayConfig(const SP & orig)
o._maintenance,
o._storeConfig,
o._threading_service_config,
+ o._alloc_config,
o._configId,
o._docTypeName);
ret->_orig = orig;
@@ -274,6 +283,7 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes)
_maintenance,
_storeConfig,
_threading_service_config,
+ _alloc_config,
_configId,
_docTypeName);
}
@@ -311,6 +321,7 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum
n._maintenance,
n._storeConfig,
n._threading_service_config,
+ n._alloc_config,
n._configId,
n._docTypeName);
result->_delayedAttributeAspects = true;
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
index dc163e91ade..8e24ed8e96a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
@@ -30,6 +30,7 @@ namespace document::internal { class InternalDocumenttypesType; }
namespace proton {
class ThreadingServiceConfig;
+class AllocConfig;
class DocumentDBConfig
{
@@ -55,6 +56,7 @@ public:
bool visibilityDelayChanged;
bool flushChanged;
bool threading_service_config_changed;
+ bool alloc_config_changed;
ComparisonResult();
ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; }
@@ -88,6 +90,7 @@ public:
return *this;
}
ComparisonResult &set_threading_service_config_changed(bool val) { threading_service_config_changed = val; return *this; }
+ ComparisonResult &set_alloc_config_changed(bool val) { alloc_config_changed = val; return *this; }
};
using SP = std::shared_ptr<DocumentDBConfig>;
@@ -131,6 +134,7 @@ private:
MaintenanceConfigSP _maintenance;
search::LogDocumentStore::Config _storeConfig;
std::shared_ptr<const ThreadingServiceConfig> _threading_service_config;
+ std::shared_ptr<const AllocConfig> _alloc_config;
SP _orig;
bool _delayedAttributeAspects;
@@ -169,6 +173,7 @@ public:
const DocumentDBMaintenanceConfig::SP &maintenance,
const search::LogDocumentStore::Config & storeConfig,
std::shared_ptr<const ThreadingServiceConfig> threading_service_config,
+ std::shared_ptr<const AllocConfig> alloc_config,
const vespalib::string &configId,
const vespalib::string &docTypeName) noexcept;
@@ -210,6 +215,8 @@ public:
bool getDelayedAttributeAspects() const { return _delayedAttributeAspects; }
const ThreadingServiceConfig& get_threading_service_config() const { return *_threading_service_config; }
const std::shared_ptr<const ThreadingServiceConfig>& get_threading_service_config_shared_ptr() const { return _threading_service_config; }
+ const AllocConfig& get_alloc_config() const { return *_alloc_config; }
+ const std::shared_ptr<const AllocConfig>& get_alloc_config_shared_ptr() const { return _alloc_config; }
bool operator==(const DocumentDBConfig &rhs) const;
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index 115a49fe997..f5a594d2f36 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -3,6 +3,7 @@
#include "documentdbconfigmanager.h"
#include "bootstrapconfig.h"
#include "threading_service_config.h"
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/config/config-ranking-constants.h>
#include <vespa/searchcore/config/config-onnx-models.h>
@@ -252,6 +253,19 @@ build_threading_service_config(const ProtonConfig &proton_config,
hw_info.cpu()));
}
+std::shared_ptr<const AllocConfig>
+build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& doc_type_name)
+{
+ auto& document_db_config_entry = find_document_db_config_entry(proton_config.documentdb, doc_type_name);
+ auto& alloc_config = document_db_config_entry.allocation;
+ auto& distribution_config = proton_config.distribution;
+ search::GrowStrategy grow_strategy(alloc_config.initialnumdocs, alloc_config.growfactor, alloc_config.growbias, alloc_config.multivaluegrowfactor);
+ search::CompactionStrategy compaction_strategy(alloc_config.maxDeadBytesRatio, alloc_config.maxDeadAddressSpaceRatio);
+ return std::make_shared<const AllocConfig>
+ (AllocStrategy(grow_strategy, compaction_strategy, alloc_config.amortizecount),
+ distribution_config.redundancy, distribution_config.searchablecopies);
+}
+
}
void
@@ -275,6 +289,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
MaintenanceConfigSP oldMaintenanceConfig;
MaintenanceConfigSP newMaintenanceConfig;
std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config;
+ std::shared_ptr<const AllocConfig> old_alloc_config;
if (!_ignoreForwardedConfig) {
if (!(_bootstrapConfig->getDocumenttypesConfigSP() &&
@@ -299,6 +314,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
newIndexschemaConfig = current->getIndexschemaConfigSP();
oldMaintenanceConfig = current->getMaintenanceConfigSP();
old_threading_service_config = current->get_threading_service_config_shared_ptr();
+ old_alloc_config = current->get_alloc_config_shared_ptr();
currentGeneration = current->getGeneration();
}
@@ -382,6 +398,10 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
(*new_threading_service_config == *old_threading_service_config)) {
new_threading_service_config = old_threading_service_config;
}
+ auto new_alloc_config = build_alloc_config(_bootstrapConfig->getProtonConfig(), _docTypeName);
+ if (new_alloc_config && old_alloc_config &&(*new_alloc_config == *old_alloc_config)) {
+ new_alloc_config = old_alloc_config;
+ }
auto newSnapshot = std::make_shared<DocumentDBConfig>(generation,
newRankProfilesConfig,
newRankingConstants,
@@ -399,6 +419,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot)
newMaintenanceConfig,
storeConfig,
new_threading_service_config,
+ new_alloc_config,
_configId,
_docTypeName);
assert(newSnapshot->valid());
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 52e5858f8a6..6ce0b896a50 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -19,16 +19,8 @@ using vespalib::makeLambdaTask;
namespace proton {
-DocumentSubDBCollection::Config::Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed,
- size_t fixedAttributeTotalSkew,
- CompactionStrategy compaction_strategy,
- size_t numSearchThreads)
- : _readyGrowth(ready),
- _notReadyGrowth(notReady),
- _removedGrowth(removed),
- _compaction_strategy(compaction_strategy),
- _fixedAttributeTotalSkew(fixedAttributeTotalSkew),
- _numSearchThreads(numSearchThreads)
+DocumentSubDBCollection::Config::Config(size_t numSearchThreads)
+ : _numSearchThreads(numSearchThreads)
{ }
DocumentSubDBCollection::DocumentSubDBCollection(
@@ -69,7 +61,6 @@ DocumentSubDBCollection::DocumentSubDBCollection(
SearchableDocSubDB::Config(
FastAccessDocSubDB::Config(
StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir,
- cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(),
_readySubDbId, SubDbType::READY),
true, true, false),
cfg.getNumSearchThreads()),
@@ -79,15 +70,14 @@ DocumentSubDBCollection::DocumentSubDBCollection(
_subDBs.push_back
(new StoreOnlyDocSubDB(
- StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, cfg.getRemovedGrowth(),
- cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(), _remSubDbId, SubDbType::REMOVED),
+ StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir,
+ _remSubDbId, SubDbType::REMOVED),
context));
_subDBs.push_back
(new FastAccessDocSubDB(
FastAccessDocSubDB::Config(
StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir,
- cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(), cfg.get_compaction_strategy(),
_notReadySubDbId, SubDbType::NOTREADY),
true, true, true),
FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService)));
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index c64d0f1a6a8..c86370c942b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -5,8 +5,6 @@
#include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
#include <vespa/searchcore/proton/common/hw_info.h>
#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
-#include <vespa/searchcommon/common/compaction_strategy.h>
-#include <vespa/searchcommon/common/growstrategy.h>
#include <vespa/searchlib/common/serialnum.h>
#include <vespa/vespalib/util/varholder.h>
#include <mutex>
@@ -62,24 +60,9 @@ public:
using SerialNum = search::SerialNum;
class Config {
public:
- using CompactionStrategy = search::CompactionStrategy;
- using GrowStrategy = search::GrowStrategy;
- Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed,
- size_t fixedAttributeTotalSkew,
- CompactionStrategy compaction_strategy,
- size_t numSearchThreads);
- GrowStrategy getReadyGrowth() const { return _readyGrowth; }
- GrowStrategy getNotReadyGrowth() const { return _notReadyGrowth; }
- GrowStrategy getRemovedGrowth() const { return _removedGrowth; }
- CompactionStrategy get_compaction_strategy() const { return _compaction_strategy; }
- size_t getNumSearchThreads() const { return _numSearchThreads; }
- size_t getFixedAttributeTotalSkew() const { return _fixedAttributeTotalSkew; }
+ Config(size_t numSearchThreads);
+ size_t getNumSearchThreads() const noexcept { return _numSearchThreads; }
private:
- const GrowStrategy _readyGrowth;
- const GrowStrategy _notReadyGrowth;
- const GrowStrategy _removedGrowth;
- const CompactionStrategy _compaction_strategy;
- const size_t _fixedAttributeTotalSkew;
const size_t _numSearchThreads;
};
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index 5bc6437b4c6..ca2a98d43e0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -13,6 +13,7 @@
#include <vespa/searchcore/proton/attribute/attribute_populator.h>
#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h>
#include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/matching/sessionmanager.h>
#include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h>
#include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h>
@@ -62,6 +63,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co
DocumentMetaStore::SP documentMetaStore,
std::shared_ptr<AttributeManager::SP> attrMgrResult) const
{
+ AllocStrategy alloc_strategy = configSnapshot.get_alloc_config().make_alloc_strategy(_subDbType);
IAttributeFactory::SP attrFactory = std::make_shared<AttributeFactory>();
AttributeManager::SP baseAttrMgr =
std::make_shared<AttributeManager>(_baseDir + "/attribute",
@@ -77,9 +79,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co
documentMetaStore,
baseAttrMgr,
(_hasAttributes ? configSnapshot.getAttributesConfig() : AttributesConfig()),
- _attributeGrow,
- _attributeGrowNumDocs,
- _attribute_compaction_strategy,
+ alloc_strategy,
_fastAccessAttributesOnly,
_writeService.master(),
attrMgrResult);
@@ -102,11 +102,10 @@ FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult)
AttributeCollectionSpec::UP
-FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, SerialNum serialNum) const
+FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const
{
uint32_t docIdLimit(_dms->getCommittedDocIdLimit());
- AttributeCollectionSpecFactory factory(_attributeGrow,
- _attributeGrowNumDocs, _attribute_compaction_strategy, _fastAccessAttributesOnly);
+ AttributeCollectionSpecFactory factory(alloc_strategy, _fastAccessAttributesOnly);
return factory.create(attrCfg, docIdLimit, serialNum);
}
@@ -247,7 +246,8 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const
{
(void) resolver;
- reconfigure(newConfigSnapshot.getStoreConfig());
+ AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType);
+ reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy);
IReprocessingTask::List tasks;
/*
* If attribute manager should change then document retriever
@@ -263,7 +263,7 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const
std::make_unique<AttributeWriterFactory>(), getSubDbName());
proton::IAttributeManager::SP oldMgr = extractAttributeManager(_fastAccessFeedView.get());
AttributeCollectionSpec::UP attrSpec =
- createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum);
+ createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum);
IReprocessingInitializer::UP initializer =
configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec);
if (initializer->hasReprocessors()) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
index 2bf574aba10..7e2ac3a67cc 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
@@ -84,7 +84,7 @@ protected:
MetricsWireService &_metricsWireService;
DocIdLimit _docIdLimit;
- AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, SerialNum serialNum) const;
+ AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const;
AttributeManager::SP getAndResetInitAttributeManager();
virtual IFlushTargetList getFlushTargetsInternal() override;
void reconfigureAttributeMetrics(const IAttributeManager &newMgr, const IAttributeManager &oldMgr);
diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
index 4fc241571ac..bb6c5423175 100644
--- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp
@@ -27,7 +27,8 @@ ReconfigParams::configHasChanged() const
_res.tuneFileDocumentDBChanged ||
_res.schemaChanged ||
_res.maintenanceChanged ||
- _res.storeChanged;
+ _res.storeChanged ||
+ _res.alloc_config_changed;
}
bool
@@ -51,7 +52,7 @@ ReconfigParams::shouldIndexManagerChange() const
bool
ReconfigParams::shouldAttributeManagerChange() const
{
- return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged;
+ return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged || _res.alloc_config_changed;
}
bool
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 2768c7ea337..51e6f8e45df 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -7,6 +7,7 @@
#include "i_document_subdb_owner.h"
#include "ibucketstatecalculator.h"
#include <vespa/searchcore/proton/attribute/attribute_writer.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/flushengine/threadedflushtarget.h>
#include <vespa/searchcore/proton/index/index_manager_initializer.h>
#include <vespa/searchcore/proton/index/index_writer.h>
@@ -143,7 +144,8 @@ IReprocessingTask::List
SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot,
SerialNum serialNum, const ReconfigParams &params, IDocumentDBReferenceResolver &resolver)
{
- StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig());
+ AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType);
+ StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy);
IReprocessingTask::List tasks;
applyFlushConfig(newConfigSnapshot.getMaintenanceConfigSP()->getFlushConfig());
if (params.shouldMatchersChange() && _addMetrics) {
@@ -152,7 +154,7 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const
if (params.shouldAttributeManagerChange()) {
proton::IAttributeManager::SP oldMgr = getAttributeManager();
AttributeCollectionSpec::UP attrSpec =
- createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum);
+ createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum);
IReprocessingInitializer::UP initializer =
_configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec, params, resolver);
if (initializer && initializer->hasReprocessors()) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index 2da0645aef1..9b504c74635 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -10,6 +10,7 @@
#include "storeonlydocsubdb.h"
#include <vespa/searchcore/proton/attribute/attribute_writer.h>
#include <vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/docsummary/summaryflushtarget.h>
#include <vespa/searchcore/proton/docsummary/summarymanagerinitializer.h>
#include <vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h>
@@ -57,15 +58,10 @@ IIndexWriter::SP nullIndexWriter;
StoreOnlyDocSubDB::Config::Config(const DocTypeName &docTypeName, const vespalib::string &subName,
const vespalib::string &baseDir,
- const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs,
- const search::CompactionStrategy& attribute_compaction_strategy,
uint32_t subDbId, SubDbType subDbType)
: _docTypeName(docTypeName),
_subName(subName),
_baseDir(baseDir + "/" + subName),
- _attributeGrow(attributeGrow),
- _attributeGrowNumDocs(attributeGrowNumDocs),
- _attribute_compaction_strategy(attribute_compaction_strategy),
_subDbId(subDbId),
_subDbType(subDbType)
{ }
@@ -102,9 +98,6 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx)
_bucketDB(ctx._bucketDB),
_bucketDBHandlerInitializer(ctx._bucketDBHandlerInitializer),
_metaStoreCtx(),
- _attributeGrow(cfg._attributeGrow),
- _attributeGrowNumDocs(cfg._attributeGrowNumDocs),
- _attribute_compaction_strategy(cfg._attribute_compaction_strategy),
_flushedDocumentMetaStoreSerialNum(0u),
_flushedDocumentStoreSerialNum(0u),
_dms(),
@@ -226,11 +219,12 @@ StoreOnlyDocSubDB::getNewestFlushedSerial()
initializer::InitializerTask::SP
StoreOnlyDocSubDB::
createSummaryManagerInitializer(const search::LogDocumentStore::Config & storeCfg,
+ const AllocStrategy& alloc_strategy,
const search::TuneFileSummary &tuneFile,
search::IBucketizer::SP bucketizer,
std::shared_ptr<SummaryManager::SP> result) const
{
- GrowStrategy grow = _attributeGrow;
+ GrowStrategy grow = alloc_strategy.get_grow_strategy();
vespalib::string baseDir(_baseDir + "/summary");
return std::make_shared<SummaryManagerInitializer>
(grow, baseDir, getSubDbName(), _docTypeName, _writeService.shared(),
@@ -249,12 +243,13 @@ StoreOnlyDocSubDB::setupSummaryManager(SummaryManager::SP summaryManager)
InitializerTask::SP
StoreOnlyDocSubDB::
-createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile,
+createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy,
+ const search::TuneFileAttributes &tuneFile,
std::shared_ptr<DocumentMetaStoreInitializerResult::SP> result) const
{
- GrowStrategy grow = _attributeGrow;
+ GrowStrategy grow = alloc_strategy.get_grow_strategy();
// Amortize memory spike cost over N docs
- grow.setDocsGrowDelta(grow.getDocsGrowDelta() + _attributeGrowNumDocs);
+ grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count());
vespalib::string baseDir(_baseDir + "/documentmetastore");
vespalib::string name = DocumentMetaStore::getFixedName();
vespalib::string attrFileName = baseDir + "/" + name; // XXX: Wrong
@@ -296,10 +291,13 @@ StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot, Ser
{
auto result = std::make_unique<DocumentSubDbInitializer>(const_cast<StoreOnlyDocSubDB &>(*this),
_writeService.master());
- auto dmsInitTask = createDocumentMetaStoreInitializer(configSnapshot.getTuneFileDocumentDBSP()->_attr,
+ AllocStrategy alloc_strategy = configSnapshot.get_alloc_config().make_alloc_strategy(_subDbType);
+ auto dmsInitTask = createDocumentMetaStoreInitializer(alloc_strategy,
+ configSnapshot.getTuneFileDocumentDBSP()->_attr,
result->writableResult().writableDocumentMetaStore());
result->addDocumentMetaStoreInitTask(dmsInitTask);
auto summaryTask = createSummaryManagerInitializer(configSnapshot.getStoreConfig(),
+ alloc_strategy,
configSnapshot.getTuneFileDocumentDBSP()->_summary,
result->result().documentMetaStore()->documentMetaStore(),
result->writableResult().writableSummaryManager());
@@ -413,14 +411,22 @@ StoreOnlyDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const
(void) params;
(void) resolver;
assert(_writeService.master().isCurrentThread());
- reconfigure(newConfigSnapshot.getStoreConfig());
+ AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType);
+ reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy);
initFeedView(newConfigSnapshot);
return IReprocessingTask::List();
}
void
-StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config)
+StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, const AllocStrategy& alloc_strategy)
{
+ auto cfg = _dms->getConfig();
+ GrowStrategy grow = alloc_strategy.get_grow_strategy();
+ // Amortize memory spike cost over N docs
+ grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count());
+ cfg.setGrowStrategy(grow);
+ cfg.setCompactionStrategy(alloc_strategy.get_compaction_strategy());
+ _dms->update_config(cfg); // Update grow and compaction config
_rSummaryMgr->reconfigure(config);
}
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index a0b278eb491..37229bd551c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -20,6 +20,7 @@
namespace proton {
+class AllocStrategy;
struct DocumentDBTaggedMetrics;
class DocumentMetaStoreInitializerResult;
class FeedHandler;
@@ -85,16 +86,11 @@ public:
const DocTypeName _docTypeName;
const vespalib::string _subName;
const vespalib::string _baseDir;
- const search::GrowStrategy _attributeGrow;
- const size_t _attributeGrowNumDocs;
- const search::CompactionStrategy _attribute_compaction_strategy;
const uint32_t _subDbId;
const SubDbType _subDbType;
Config(const DocTypeName &docTypeName, const vespalib::string &subName,
const vespalib::string &baseDir,
- const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs,
- const search::CompactionStrategy& attribute_compaction_strategy,
uint32_t subDbId, SubDbType subDbType);
~Config();
};
@@ -133,9 +129,6 @@ protected:
BucketDBOwner::SP _bucketDB;
bucketdb::IBucketDBHandlerInitializer &_bucketDBHandlerInitializer;
IDocumentMetaStoreContext::SP _metaStoreCtx;
- const search::GrowStrategy _attributeGrow;
- const size_t _attributeGrowNumDocs;
- const search::CompactionStrategy _attribute_compaction_strategy;
// The following two serial numbers reflect state at program startup
// and are used by replay logic.
SerialNum _flushedDocumentMetaStoreSerialNum;
@@ -168,6 +161,7 @@ protected:
std::shared_ptr<initializer::InitializerTask>
createSummaryManagerInitializer(const search::LogDocumentStore::Config & protonSummaryCfg,
+ const AllocStrategy& alloc_strategy,
const search::TuneFileSummary &tuneFile,
search::IBucketizer::SP bucketizer,
std::shared_ptr<SummaryManager::SP> result) const;
@@ -175,7 +169,8 @@ protected:
void setupSummaryManager(SummaryManager::SP summaryManager);
std::shared_ptr<initializer::InitializerTask>
- createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile,
+ createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy,
+ const search::TuneFileAttributes &tuneFile,
std::shared_ptr<std::shared_ptr<DocumentMetaStoreInitializerResult>> result) const;
void setupDocumentMetaStore(std::shared_ptr<DocumentMetaStoreInitializerResult> dmsResult);
@@ -185,7 +180,8 @@ protected:
StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams();
vespalib::string getSubDbName() const;
- void reconfigure(const search::LogDocumentStore::Config & protonConfig);
+ void reconfigure(const search::LogDocumentStore::Config & protonConfig,
+ const AllocStrategy& alloc_strategy);
public:
StoreOnlyDocSubDB(const Config &cfg, const Context &ctx);
~StoreOnlyDocSubDB() override;
diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp
index 335c7aca24c..17cb91ac4ce 100644
--- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp
+++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp
@@ -10,6 +10,7 @@
#include <vespa/searchsummary/config/config-juniperrc.h>
#include <vespa/document/config/config-documenttypes.h>
#include <vespa/config-imported-fields.h>
+#include <vespa/searchcore/proton/common/alloc_config.h>
#include <vespa/searchcore/proton/server/threading_service_config.h>
using document::DocumenttypesConfig;
@@ -47,6 +48,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation,
_maintenance(std::make_shared<DocumentDBMaintenanceConfig>()),
_store(),
_threading_service_config(std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1))),
+ _alloc_config(std::make_shared<const AllocConfig>()),
_configId(configId),
_docTypeName(docTypeName)
{ }
@@ -70,6 +72,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg)
_maintenance(cfg.getMaintenanceConfigSP()),
_store(cfg.getStoreConfig()),
_threading_service_config(cfg.get_threading_service_config_shared_ptr()),
+ _alloc_config(cfg.get_alloc_config_shared_ptr()),
_configId(cfg.getConfigId()),
_docTypeName(cfg.getDocTypeName())
{}
@@ -97,6 +100,7 @@ DocumentDBConfigBuilder::build()
_maintenance,
_store,
_threading_service_config,
+ _alloc_config,
_configId,
_docTypeName);
}
diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h
index 218a7c56fa9..706e14e73db 100644
--- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h
+++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h
@@ -28,6 +28,7 @@ private:
DocumentDBConfig::MaintenanceConfigSP _maintenance;
search::LogDocumentStore::Config _store;
std::shared_ptr<const ThreadingServiceConfig> _threading_service_config;
+ std::shared_ptr<const AllocConfig> _alloc_config;
vespalib::string _configId;
vespalib::string _docTypeName;
diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
index dde6b14121d..36a697eaa12 100644
--- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
+++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp
@@ -226,4 +226,24 @@ TEST_F("Compaction is not executed when free lists are used",
EXPECT_EQUAL(1001u, afterSpace.dead());
}
+TEST_F("Compaction is peformed when compaction strategy is changed to enable compaction",
+ Fixture(compactAddressSpaceAttributeConfig(false)))
+{
+ populate_and_hammer(f, true);
+ AddressSpace after1 = f.getMultiValueAddressSpaceUsage("after1");
+ // 100 * 1000 dead arrays due to new values for docids
+ // 1 reserved array accounted as dead
+ EXPECT_EQUAL(100001u, after1.dead());
+ f._v->update_config(compactAddressSpaceAttributeConfig(true));
+ auto old_dead = after1.dead();
+ AddressSpace after2 = f.getMultiValueAddressSpaceUsage("after2");
+ while (after2.dead() < old_dead) {
+ old_dead = after2.dead();
+ f._v->commit(); // new commit might trigger further compaction
+ after2 = f.getMultiValueAddressSpaceUsage("after2");
+ }
+ // DEAD_ARRAYS_SLACK in multi value mapping is is 64k
+ EXPECT_GREATER(65536u, after2.dead());
+}
+
TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
index 2168bbe4276..d2574bd32a2 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
@@ -21,6 +21,7 @@
#include <vespa/vespalib/util/exceptions.h>
#include <vespa/searchlib/util/logutil.h>
#include <vespa/searchcommon/attribute/attribute_utils.h>
+#include <thread>
#include <vespa/log/log.h>
LOG_SETUP(".searchlib.attribute.attributevector");
@@ -769,6 +770,31 @@ AttributeVector::logEnumStoreEvent(const char *reason, const char *stage)
EV_STATE(eventName.c_str(), jstr.toString().data());
}
+void
+AttributeVector::drain_hold(uint64_t hold_limit)
+{
+ incGeneration();
+ for (int retry = 0; ; ++retry) {
+ removeAllOldGenerations();
+ updateStat(true);
+ if (_status.getOnHold() <= hold_limit) {
+ return;
+ }
+ std::this_thread::sleep_for(retry < 20 ? 20ms : 100ms);
+ }
+}
+
+void
+AttributeVector::update_config(const Config& cfg)
+{
+ commit(true);
+ drain_hold(1024 * 1024); // Wait until 1MiB or less on hold
+ _config.setGrowStrategy(cfg.getGrowStrategy());
+ _config.setCompactionStrategy(cfg.getCompactionStrategy());
+ commit(); // might trigger compaction if compaction strategy changed
+ drain_hold(1024 * 1024); // Wait until 1MiB or less on hold
+}
+
template bool AttributeVector::append<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t, bool);
template bool AttributeVector::update<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &);
template bool AttributeVector::remove<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t);
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h
index 4fc6589850e..f308ee8d024 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h
@@ -393,6 +393,7 @@ public:
/** Return the fixed length of the attribute. If 0 then you must inquire each document. */
size_t getFixedWidth() const override { return _config.basicType().fixedSize(); }
const Config &getConfig() const { return _config; }
+ void update_config(const Config& cfg);
BasicType getInternalBasicType() const { return _config.basicType(); }
CollectionType getInternalCollectionType() const { return _config.collectionType(); }
const BaseName & getBaseFileName() const { return _baseFileName; }
@@ -667,6 +668,8 @@ public:
static bool isEnumerated(const vespalib::GenericHeader &header);
virtual vespalib::MemoryUsage getChangeVectorMemoryUsage() const;
+
+ void drain_hold(uint64_t hold_limit);
};
}