diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-13 10:33:06 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-08-13 10:33:06 +0200 |
commit | 1f49eabb261f4db4a47ec47d1971d4927cab867f (patch) | |
tree | 70e5237fb73331555e2f4c1f9b916c51b457b621 | |
parent | f9f97c0e1f02c9a17976ce89da778e63e02459de (diff) | |
parent | 48bd9dbe34bff8138b3610169a0c5acc226e06c9 (diff) |
Merge pull request #18733 from vespa-engine/balder/provide-the-shared-executor-while-loading
Wire in the shared executor for use when loading attributes.
4 files changed, 18 insertions, 11 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index 264d6d397e1..a03ecdff1e0 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> LOG_SETUP("attribute_initializer_test"); @@ -89,6 +90,7 @@ struct Fixture DirectoryHandler _dirHandler; std::shared_ptr<AttributeDiskLayout> _diskLayout; AttributeFactory _factory; + vespalib::ThreadStackExecutor _executor; Fixture(); ~Fixture(); std::unique_ptr<AttributeInitializer> createInitializer(const AttributeSpec &spec, SerialNum serialNum); @@ -97,7 +99,8 @@ struct Fixture Fixture::Fixture() : _dirHandler(test_dir), _diskLayout(AttributeDiskLayout::create(test_dir)), - _factory() + _factory(), + _executor(1, 0x10000) { } @@ -106,7 +109,7 @@ Fixture::~Fixture() = default; std::unique_ptr<AttributeInitializer> Fixture::createInitializer(const AttributeSpec &spec, SerialNum serialNum) { - return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(spec.getName()), "test.subdb", spec, serialNum, _factory); + return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(spec.getName()), "test.subdb", spec, serialNum, _factory, _executor); } TEST("require that integer attribute can be initialized") diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 4cfa3dd8b13..c1ae0a41a6f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -8,7 +8,6 @@ #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/searchcommon/attribute/persistent_predicate_params.h> #include <vespa/searchlib/util/fileutil.h> #include <vespa/searchlib/attribute/attribute_header.h> @@ -194,7 +193,7 @@ AttributeInitializer::loadAttribute(const AttributeVectorSP &attr, assert(attr->hasLoadData()); vespalib::Timer timer; EventLogger::loadAttributeStart(_documentSubDbName, attr->getName()); - if (!attr->load()) { + if (!attr->load(&_executor)) { LOG(warning, "Could not load attribute vector '%s' from disk. Returning empty attribute vector", attr->getBaseFileName().c_str()); return false; @@ -235,12 +234,14 @@ AttributeInitializer::AttributeInitializer(const std::shared_ptr<AttributeDirect const vespalib::string &documentSubDbName, const AttributeSpec &spec, uint64_t currentSerialNum, - const IAttributeFactory &factory) + const IAttributeFactory &factory, + vespalib::Executor & executor) : _attrDir(attrDir), _documentSubDbName(documentSubDbName), _spec(spec), _currentSerialNum(currentSerialNum), _factory(factory), + _executor(executor), _header(), _header_ok(false) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h index a2277fd8134..b8c494394d7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h @@ -8,6 +8,7 @@ #include <vespa/searchlib/common/serialnum.h> namespace search::attribute { class AttributeHeader; } +namespace vespalib { class Executor; } namespace proton { @@ -29,6 +30,7 @@ private: const AttributeSpec _spec; const uint64_t _currentSerialNum; const IAttributeFactory &_factory; + vespalib::Executor &_executor; std::unique_ptr<const search::attribute::AttributeHeader> _header; bool _header_ok; @@ -45,7 +47,8 @@ private: public: AttributeInitializer(const std::shared_ptr<AttributeDirectory> &attrDir, const vespalib::string &documentSubDbName, - const AttributeSpec &spec, uint64_t currentSerialNum, const IAttributeFactory &factory); + const AttributeSpec &spec, uint64_t currentSerialNum, const IAttributeFactory &factory, + vespalib::Executor & executor); ~AttributeInitializer(); AttributeInitializerResult init() const; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index f635ee34a04..f9a5624c5c8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -19,7 +19,7 @@ #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/searchlib/common/threaded_compactable_lid_space.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/threadexecutor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> @@ -120,7 +120,7 @@ AttributeManager::internalAddAttribute(const AttributeSpec &spec, uint64_t serialNum, const IAttributeFactory &factory) { - AttributeInitializer initializer(_diskLayout->createAttributeDir(spec.getName()), _documentSubDbName, spec, serialNum, factory); + AttributeInitializer initializer(_diskLayout->createAttributeDir(spec.getName()), _documentSubDbName, spec, serialNum, factory, _shared_executor); AttributeInitializerResult result = initializer.init(); if (result) { result.getAttribute()->setInterlock(_interlock); @@ -196,9 +196,9 @@ AttributeManager::addNewAttributes(const Spec &newSpec, LOG(debug, "Creating initializer for attribute vector '%s': docIdLimit=%u, serialNumber=%" PRIu64, aspec.getName().c_str(), newSpec.getDocIdLimit(), newSpec.getCurrentSerialNum()); - AttributeInitializer::UP initializer = - std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(aspec.getName()), _documentSubDbName, - aspec, newSpec.getCurrentSerialNum(), *_factory); + auto initializer = std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(aspec.getName()), + _documentSubDbName, aspec, newSpec.getCurrentSerialNum(), + *_factory, _shared_executor); initializerRegistry.add(std::move(initializer)); // TODO: Might want to use hardlinks to make attribute vector |