diff options
74 files changed, 919 insertions, 360 deletions
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp index ac7fe34e06b..e7a4c28b27a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp @@ -16,13 +16,9 @@ Config::Config() : _enableOnlyBitVector(false), _isFilter(false), _fastAccess(false), - _maxInternalBlobSize(defaultMaxInternalBlobSize), _growStrategy(), _compactionStrategy(), - _arity(8), - _lower_bound(LLONG_MIN), - _upper_bound(LLONG_MAX), - _dense_posting_list_threshold(0.4), + _predicateParams(), _tensorType(vespalib::eval::ValueType::error_type()) { } @@ -39,13 +35,9 @@ Config::Config(BasicType bt, _enableOnlyBitVector(false), _isFilter(false), _fastAccess(false), - _maxInternalBlobSize(defaultMaxInternalBlobSize), _growStrategy(), _compactionStrategy(), - _arity(8), - _lower_bound(LLONG_MIN), - _upper_bound(LLONG_MAX), - _dense_posting_list_threshold(0.4), + _predicateParams(), _tensorType(vespalib::eval::ValueType::error_type()) { } diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index d0b2535950f..be2a1d2777d 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -7,12 +7,11 @@ #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchcommon/common/compaction_strategy.h> #include <vespa/eval/eval/value_type.h> +#include "predicate_params.h" namespace search { namespace attribute { -const size_t defaultMaxInternalBlobSize = 0x400000000ul; - class Config { public: @@ -27,11 +26,7 @@ public: CollectionType collectionType() const { return _type; } bool fastSearch() const { return _fastSearch; } bool huge() const { return _huge; } - size_t getMaxInternalBlobSize() const { return _maxInternalBlobSize; } - uint32_t arity() const { return _arity; } - int64_t lower_bound() const { return _lower_bound; } - int64_t upper_bound() const { return _upper_bound; } - double dense_posting_list_threshold() const { return _dense_posting_list_threshold; } + const PredicateParams &predicateParams() const { return _predicateParams; } vespalib::eval::ValueType tensorType() const { return _tensorType; } /** @@ -70,11 +65,7 @@ public: const CompactionStrategy &getCompactionStrategy() const { return _compactionStrategy; } void setHuge(bool v) { _huge = v; } void setFastSearch(bool v) { _fastSearch = v; } - void setMaxInternalBlobSize(size_t v) { _maxInternalBlobSize = v; } - void setArity(uint32_t v) { _arity = v; } - void setBounds(int64_t lower, int64_t upper) { _lower_bound = lower; - _upper_bound = upper; } - void setDensePostingListThreshold(double v) { _dense_posting_list_threshold = v; } + void setPredicateParams(const PredicateParams &v) { _predicateParams = v; } void setTensorType(const vespalib::eval::ValueType &tensorType_in) { _tensorType = tensorType_in; } @@ -126,13 +117,9 @@ public: _enableOnlyBitVector == b._enableOnlyBitVector && _isFilter == b._isFilter && _fastAccess == b._fastAccess && - _maxInternalBlobSize == b._maxInternalBlobSize && _growStrategy == b._growStrategy && _compactionStrategy == b._compactionStrategy && - _arity == b._arity && - _lower_bound == b._lower_bound && - _upper_bound == b._upper_bound && - _dense_posting_list_threshold == b._dense_posting_list_threshold && + _predicateParams == b._predicateParams && (_basicType.type() != BasicType::Type::TENSOR || _tensorType == b._tensorType); } @@ -146,13 +133,9 @@ private: bool _enableOnlyBitVector; bool _isFilter; bool _fastAccess; - size_t _maxInternalBlobSize; GrowStrategy _growStrategy; CompactionStrategy _compactionStrategy; - uint32_t _arity; - int64_t _lower_bound; - int64_t _upper_bound; - double _dense_posting_list_threshold; + PredicateParams _predicateParams; vespalib::eval::ValueType _tensorType; }; } // namespace attribute diff --git a/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h b/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h new file mode 100644 index 00000000000..1aed2dad72c --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h @@ -0,0 +1,37 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search { +namespace attribute { + +/* + * Persistent parameters for predicate attributes. + */ +class PersistentPredicateParams { + uint32_t _arity; + int64_t _lower_bound; + int64_t _upper_bound; + +public: + PersistentPredicateParams() + : _arity(8), + _lower_bound(std::numeric_limits<int64_t>::min()), + _upper_bound(std::numeric_limits<int64_t>::max()) + { + } + uint32_t arity() const { return _arity; } + int64_t lower_bound() const { return _lower_bound; } + int64_t upper_bound() const { return _upper_bound; } + void setArity(uint32_t v) { _arity = v; } + void setBounds(int64_t lower, int64_t upper) { _lower_bound = lower; _upper_bound = upper; } + + bool operator==(const PersistentPredicateParams &rhs) const { + return ((_arity == rhs._arity) && + (_lower_bound == rhs._lower_bound) && + (_upper_bound == rhs._upper_bound)); + } +}; + +} // namespace attribute +} // namespace search diff --git a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h new file mode 100644 index 00000000000..8e8e0605b17 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "persistent_predicate_params.h" + +namespace search { +namespace attribute { + +/* + * Parameters for predicate attributes. + */ +class PredicateParams : public PersistentPredicateParams +{ + double _dense_posting_list_threshold; +public: + PredicateParams() + : PersistentPredicateParams(), + _dense_posting_list_threshold(0.4) + { + } + + double dense_posting_list_threshold() const { return _dense_posting_list_threshold; } + void setDensePostingListThreshold(double v) { _dense_posting_list_threshold = v; } + bool operator==(const PredicateParams &rhs) const { + return (PersistentPredicateParams::operator==(rhs) && + (_dense_posting_list_threshold == rhs._dense_posting_list_threshold)); + } +}; + +} // namespace attribute +} // namespace search diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 7c96a567f55..d7edca0ac35 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -62,6 +62,7 @@ vespa_define_module( src/tests/grouping src/tests/proton/attribute src/tests/proton/attribute/attribute_directory + src/tests/proton/attribute/attribute_initializer src/tests/proton/attribute/attribute_manager src/tests/proton/attribute/attribute_populator src/tests/proton/attribute/attribute_usage_filter diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt new file mode 100644 index 00000000000..ad1383f6247 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_initializer_test_app TEST + SOURCES + attribute_initializer_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attribute_initializer_test_app COMMAND searchcore_attribute_initializer_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/DESC b/searchcore/src/tests/proton/attribute/attribute_initializer/DESC new file mode 100644 index 00000000000..d7e2c61d85d --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/DESC @@ -0,0 +1 @@ +attribute initializer test. Take a look at attribute_initializer_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/FILES b/searchcore/src/tests/proton/attribute/attribute_initializer/FILES new file mode 100644 index 00000000000..cedbcb0155b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/FILES @@ -0,0 +1 @@ +attribute_initializer_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp new file mode 100644 index 00000000000..37ef21fb700 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -0,0 +1,182 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_initializer_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/attribute/attribute_directory.h> +#include <vespa/searchcore/proton/attribute/attribute_factory.h> +#include <vespa/searchcore/proton/attribute/attribute_initializer.h> + +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::SerialNum; + +const vespalib::string test_dir = "test_output"; + +namespace proton +{ + +namespace { + +const Config int32_sv(BasicType::Type::INT32); +const Config int16_sv(BasicType::Type::INT16); +const Config int32_array(BasicType::Type::INT32, CollectionType::Type::ARRAY); +const Config predicate(BasicType::Type::PREDICATE); + +Config getPredicateWithArity(uint32_t arity) +{ + Config ret(predicate); + search::attribute::PredicateParams predicateParams; + predicateParams.setArity(arity); + ret.setPredicateParams(predicateParams); + return ret; +} + +Config getTensor(const vespalib::string &spec) +{ + Config ret(BasicType::Type::TENSOR); + ret.setTensorType(vespalib::eval::ValueType::from_spec(spec)); + return ret; +} + +void +saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, SerialNum createSerialNum) +{ + auto diskLayout = AttributeDiskLayout::create(test_dir); + auto dir = diskLayout->createAttributeDir(name); + auto writer = dir->getWriter(); + writer->createInvalidSnapshot(serialNum); + auto snapshotdir = writer->getSnapshotDir(serialNum); + vespalib::mkdir(snapshotdir); + auto av = search::AttributeFactory::createAttribute(snapshotdir + "/" + name, cfg); + av->setCreateSerialNum(createSerialNum); + av->addReservedDoc(); + uint32_t docId; + av->addDoc(docId); + assert(docId == 1u); + av->clearDoc(docId); + av->save(); + writer->markValidSnapshot(serialNum); +} + +} + +struct Fixture +{ + test::DirectoryHandler _dirHandler; + std::shared_ptr<AttributeDiskLayout> _diskLayout; + AttributeFactory _factory; + Fixture(); + ~Fixture(); + std::unique_ptr<AttributeInitializer> createInitializer(const vespalib::string &name, const Config &cfg, SerialNum serialNum); +}; + +Fixture::Fixture() + : _dirHandler(test_dir), + _diskLayout(AttributeDiskLayout::create(test_dir)), + _factory() +{ +} + +Fixture::~Fixture() {} + +std::unique_ptr<AttributeInitializer> +Fixture::createInitializer(const vespalib::string &name, const Config &cfg, SerialNum serialNum) +{ + return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(name), "test.subdb", cfg, serialNum, _factory); +} + +TEST("require that integer attribute can be initialized") +{ + saveAttr("a", int32_sv, 10, 2); + Fixture f; + auto av = f.createInitializer("a", int32_sv, 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that mismatching base type is not loaded") +{ + saveAttr("a", int32_sv, 10, 2); + Fixture f; + auto av = f.createInitializer("a", int16_sv, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that mismatching collection type is not loaded") +{ + saveAttr("a", int32_sv, 10, 2); + Fixture f; + auto av = f.createInitializer("a", int32_array, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that predicate attributes can be initialized") +{ + saveAttr("a", predicate, 10, 2); + Fixture f; + auto av = f.createInitializer("a", predicate, 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that predicate attributes will not be initialized with future-created attribute") +{ + saveAttr("a", predicate, 10, 8); + Fixture f; + auto av = f.createInitializer("a", predicate, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that predicate attributes will not be initialized with mismatching type") +{ + saveAttr("a", predicate, 10, 2); + Fixture f; + auto av = f.createInitializer("a", getPredicateWithArity(4), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that tensor attribute can be initialized") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 2); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[10])"), 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that tensor attributes will not be initialized with future-created attribute") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 8); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[10])"), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that tensor attributes will not be initialized with mismatching type") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 2); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[11])"), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +} + +TEST_MAIN() +{ + vespalib::rmdir(test_dir, true); + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index a030c65954d..7232f997b0e 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -61,6 +61,7 @@ using search::predicate::PredicateIndex; using search::predicate::PredicateTreeAnnotations; using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; +using vespalib::eval::ValueType; typedef search::attribute::Config AVConfig; typedef proton::AttributeCollectionSpec::Attribute AttrSpec; @@ -111,6 +112,19 @@ fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64 test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); } +search::SerialNum getCreateSerialNum(const AttributeGuard::UP &guard) +{ + if (!guard || !guard->valid()) { + return 0; + } else { + return (*guard)->getCreateSerialNum(); + } +} + +void assertCreateSerialNum(const AttributeManager &am, const vespalib::string &name, search::SerialNum expCreateSerialNum) { + EXPECT_EQUAL(expCreateSerialNum, getCreateSerialNum(am.getAttribute(name))); +} + struct ImportedAttributesRepoBuilder { ImportedAttributesRepo::UP _repo; ImportedAttributesRepoBuilder() : _repo(std::make_unique<ImportedAttributesRepo>()) {} @@ -740,6 +754,53 @@ TEST_F("require that imported attributes are exposed via attribute context toget EXPECT_EQUAL("imported", all[1]->getName()); } +TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) +{ + AVConfig generic_tensor(BasicType::TENSOR); + generic_tensor.setTensorType(ValueType::tensor_type({})); + AVConfig dense_tensor(BasicType::TENSOR); + dense_tensor.setTensorType(ValueType::from_spec("tensor(x[10])")); + AVConfig predicate(BasicType::PREDICATE); + using PredicateParams = search::attribute::PredicateParams; + PredicateParams predicateParams; + predicateParams.setArity(2); + predicate.setPredicateParams(predicateParams); + AVConfig predicate2(BasicType::PREDICATE); + PredicateParams predicateParams2; + predicateParams2.setArity(4); + predicate2.setPredicateParams(predicateParams2); + + auto am1(std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), + f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + am1->addAttribute("a1", INT32_SINGLE, 1); + am1->addAttribute("a2", INT32_SINGLE, 2); + am1->addAttribute("a3", generic_tensor, 3); + am1->addAttribute("a4", generic_tensor, 4); + am1->addAttribute("a5", predicate, 5); + am1->addAttribute("a6", predicate, 6); + AttrSpecList newSpec; + newSpec.push_back(AttrSpec("a1", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a2", INT32_ARRAY)); + newSpec.push_back(AttrSpec("a3", generic_tensor)); + newSpec.push_back(AttrSpec("a4", dense_tensor)); + newSpec.push_back(AttrSpec("a5", predicate)); + newSpec.push_back(AttrSpec("a6", predicate2)); + SequentialAttributeManager am2(*am1, AttrMgrSpec(newSpec, 5, 20)); + TEST_DO(assertCreateSerialNum(*am1, "a1", 1)); + TEST_DO(assertCreateSerialNum(*am1, "a2", 2)); + TEST_DO(assertCreateSerialNum(*am1, "a3", 3)); + TEST_DO(assertCreateSerialNum(*am1, "a4", 4)); + TEST_DO(assertCreateSerialNum(*am1, "a5", 5)); + TEST_DO(assertCreateSerialNum(*am1, "a6", 6)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a1", 1)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a2", 20)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a3", 3)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a4", 20)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a5", 5)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a6", 20)); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index e28dd24a454..19bfedfcd31 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -68,7 +68,7 @@ struct Fixture _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), _fileHeader, _attributeFieldWriter, _hwInfo)), - _pop(_mgr, 1, "test"), + _pop(_mgr, 1, "test", CREATE_SERIAL_NUM), _ctx() { _mgr->addAttribute("a1", AVConfig(AVBasicType::INT32), @@ -93,6 +93,8 @@ TEST_F("require that reprocess with document populates attribute", Fixture) EXPECT_EQUAL(7u, attr->get()->getNumDocs()); EXPECT_EQUAL(44, attr->get()->getInt(6)); EXPECT_EQUAL(2u, attr->get()->getStatus().getLastSyncToken()); + f._pop.done(); + EXPECT_EQUAL(CREATE_SERIAL_NUM, attr->get()->getStatus().getLastSyncToken()); } TEST_MAIN() diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 36319ae77d5..f560030db97 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -5,12 +5,15 @@ LOG_SETUP("attribute_reprocessing_initializer_test"); #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcore/proton/test/directory_handler.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -19,6 +22,8 @@ LOG_SETUP("attribute_reprocessing_initializer_test"); using namespace proton; using namespace search; using namespace search::index; +using search::attribute::Config; +using search::attribute::BasicType; const vespalib::string TEST_DIR = "test_output"; const SerialNum INIT_SERIAL_NUM = 10; @@ -118,10 +123,11 @@ struct Fixture _handler() { } + ~Fixture() { } void init() { _initializer.reset(new AttributeReprocessingInitializer (ARIConfig(_newCfg._mgr, _newCfg._schema, _newCfg._inspector), - ARIConfig(_oldCfg._mgr, _oldCfg._schema, _oldCfg._inspector), "test")); + ARIConfig(_oldCfg._mgr, _oldCfg._schema, _oldCfg._inspector), "test", INIT_SERIAL_NUM)); _initializer->initialize(_handler); } Fixture &addOldConfig(const StringVector &fields, @@ -249,6 +255,25 @@ TEST_F("require that predicate fields are not populated from attribute", Fixture EXPECT_TRUE(f.assertFields({"c", "d"})); } +TEST("require that added attribute aspect with flushed attribute after interruptted reprocessing does not require attribute populate") +{ + { + auto diskLayout = AttributeDiskLayout::create(TEST_DIR); + auto dir = diskLayout->createAttributeDir("a"); + auto writer = dir->getWriter(); + writer->createInvalidSnapshot(INIT_SERIAL_NUM); + writer->markValidSnapshot(INIT_SERIAL_NUM); + auto snapshotdir = writer->getSnapshotDir(INIT_SERIAL_NUM); + vespalib::mkdir(snapshotdir); + auto av = AttributeFactory::createAttribute(snapshotdir + "/a", + Config(BasicType::STRING)); + av->save(); + } + Fixture f; + f.addOldConfig({"a"}, {}).addNewConfig({"a"}, {"a"}).init(); + EXPECT_TRUE(f.assertAttributes({})); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp index 483c1c3b977..1848ad8fc74 100644 --- a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp @@ -23,6 +23,7 @@ struct MyProcessor : public ReprocessingType _lid = lid; _docId = doc.getId(); } + virtual void done() { } }; typedef MyProcessor<IReprocessingReader, const Document &> MyReader; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 1b998646a25..7a23db1a071 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -7,64 +7,86 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/searchlib/util/fileutil.h> +#include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_initializer"); +using search::attribute::BasicType; using search::attribute::Config; using search::AttributeVector; using search::IndexMetaInfo; namespace proton { -typedef AttributeInitializer::AttributeHeader AttributeHeader; +using search::attribute::AttributeHeader; namespace { -const vespalib::string dataTypeTag = "datatype"; -const vespalib::string collectionTypeTag = "collectiontype"; -const vespalib::string createSerialNumTag = "createSerialNum"; +vespalib::string +extraPredicateType(const search::attribute::PersistentPredicateParams ¶ms) +{ + vespalib::asciistream os; + os << "arity=" << params.arity(); + os << ",lower_bound=" << params.lower_bound(); + os << ",upper_bound=" << params.upper_bound(); + return os.str(); +} -uint64_t -extractCreateSerialNum(const vespalib::GenericHeader &header) +vespalib::string +extraType(const Config &cfg) { - if (header.hasTag(createSerialNumTag)) { - return header.getTag(createSerialNumTag).asInteger(); - } else { - return 0u; + if (cfg.basicType().type() == BasicType::Type::TENSOR) { + return cfg.tensorType().to_spec(); + } + if (cfg.basicType().type() == BasicType::Type::PREDICATE) { + return extraPredicateType(cfg.predicateParams()); + } + return ""; +} + +vespalib::string +extraType(const AttributeHeader &header) +{ + if (header.getBasicType().type() == BasicType::Type::TENSOR) { + return header.getTensorType().to_spec(); + } + if (header.getBasicType().type() == BasicType::Type::PREDICATE) { + return extraPredicateType(header.getPredicateParams()); } + return ""; } bool -extractHeaderTypeOK(const vespalib::GenericHeader &header, const Config &cfg) +headerTypeOK(const AttributeHeader &header, const Config &cfg) { - return header.hasTag(dataTypeTag) && - header.hasTag(collectionTypeTag) && - header.getTag(dataTypeTag).asString() == - cfg.basicType().asString() && - header.getTag(collectionTypeTag).asString() == - cfg.collectionType().asString(); + if ((header.getBasicType().type() != cfg.basicType().type()) || + (header.getCollectionType().type() != cfg.collectionType().type())) { + return false; + } + if (cfg.basicType().type() == BasicType::Type::TENSOR) { + if (header.getTensorType() != cfg.tensorType()) { + return false; + } + } + if (cfg.basicType().type() == BasicType::PREDICATE) { + if (header.getPredicateParamsSet()) { + if (!(header.getPredicateParams() == cfg.predicateParams())) { + return false; + } + } + } + return true; } AttributeHeader -extractHeader(const AttributeVector::SP &attr, - const vespalib::string &attrFileName) +extractHeader(const vespalib::string &attrFileName) { - auto df = search::FileUtil::openFile(attrFileName + ".dat"); vespalib::FileHeader datHeader; datHeader.readFile(*df); - AttributeHeader retval; - retval._createSerialNum = extractCreateSerialNum(datHeader); - retval._headerTypeOK = extractHeaderTypeOK(datHeader, attr->getConfig()); - if (datHeader.hasTag(dataTypeTag)) { - retval._btString = datHeader.getTag(dataTypeTag).asString(); - } - if (datHeader.hasTag(collectionTypeTag)) { - retval._ctString = datHeader.getTag(collectionTypeTag).asString(); - } - return retval; + return AttributeHeader::extractTags(datHeader); } void @@ -73,9 +95,9 @@ logAttributeTooNew(const AttributeVector::SP &attr, uint64_t serialNum) { LOG(info, "Attribute vector '%s' is too new (%" PRIu64 " > %" PRIu64 ")", - attr->getBaseFileName().c_str(), - header._createSerialNum, - serialNum); + attr->getBaseFileName().c_str(), + header.getCreateSerialNum(), + serialNum); } void @@ -83,26 +105,20 @@ logAttributeWrongType(const AttributeVector::SP &attr, const AttributeHeader &header) { const Config &cfg(attr->getConfig()); - LOG(info, "Attribute vector '%s' is of wrong type (expected %s/%s, got %s/%s)", + vespalib::string extraCfgType = extraType(cfg); + vespalib::string extraHeaderType = extraType(header); + LOG(info, "Attribute vector '%s' is of wrong type (expected %s/%s/%s, got %s/%s/%s)", attr->getBaseFileName().c_str(), cfg.basicType().asString(), cfg.collectionType().asString(), - header._btString.c_str(), - header._ctString.c_str()); -} - + extraCfgType.c_str(), + header.getBasicType().asString(), + header.getCollectionType().asString(), + extraHeaderType.c_str()); } -AttributeInitializer::AttributeHeader::AttributeHeader() - : _createSerialNum(0), - _headerTypeOK(false), - _btString("unknown"), - _ctString("unknown") -{ } -AttributeInitializer::AttributeHeader::~AttributeHeader() {} - AttributeVector::SP AttributeInitializer::tryLoadAttribute() const { @@ -110,8 +126,8 @@ AttributeInitializer::tryLoadAttribute() const vespalib::string attrFileName = _attrDir->getAttributeFileName(serialNum); AttributeVector::SP attr = _factory.create(attrFileName, _cfg); if (serialNum != 0) { - AttributeHeader header = extractHeader(attr, attrFileName); - if (header._createSerialNum > _currentSerialNum || !header._headerTypeOK) { + AttributeHeader header = extractHeader(attrFileName); + if (header.getCreateSerialNum() > _currentSerialNum || !headerTypeOK(header, attr->getConfig())) { setupEmptyAttribute(attr, serialNum, header); return attr; } @@ -150,10 +166,10 @@ AttributeInitializer::setupEmptyAttribute(AttributeVector::SP &attr, search::SerialNum serialNum, const AttributeHeader &header) const { - if (header._createSerialNum > _currentSerialNum) { + if (header.getCreateSerialNum() > _currentSerialNum) { logAttributeTooNew(attr, header, _currentSerialNum); } - if (!header._headerTypeOK) { + if (!headerTypeOK(header, attr->getConfig())) { logAttributeWrongType(attr, header); } LOG(info, "Returning empty attribute vector for '%s'", diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h index d396c663710..8643fd4c7df 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h @@ -6,6 +6,9 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchcommon/attribute/persistent_predicate_params.h> + +namespace search { namespace attribute { class AttributeHeader; } } namespace proton { @@ -19,16 +22,6 @@ class AttributeInitializer public: typedef std::unique_ptr<AttributeInitializer> UP; - struct AttributeHeader - { - uint64_t _createSerialNum; - bool _headerTypeOK; - vespalib::string _btString; - vespalib::string _ctString; - AttributeHeader(); - ~AttributeHeader(); - }; - private: std::shared_ptr<AttributeDirectory> _attrDir; const vespalib::string _documentSubDbName; @@ -43,7 +36,7 @@ private: void setupEmptyAttribute(search::AttributeVector::SP &attr, search::SerialNum serialNum, - const AttributeHeader &header) const; + const search::attribute::AttributeHeader &header) const; search::AttributeVector::SP createAndSetupEmptyAttribute() const; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp index 226a79b6d49..dadf5d41912 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp @@ -15,6 +15,7 @@ namespace proton { search::SerialNum AttributePopulator::nextSerialNum() { + assert(_currSerialNum <= _configSerialNum); return _currSerialNum++; } @@ -32,10 +33,12 @@ AttributePopulator::getNames() const AttributePopulator::AttributePopulator(const proton::IAttributeManager::SP &mgr, search::SerialNum initSerialNum, - const vespalib::string &subDbName) + const vespalib::string &subDbName, + search::SerialNum configSerialNum) : _writer(mgr), _initSerialNum(initSerialNum), _currSerialNum(initSerialNum), + _configSerialNum(configSerialNum), _subDbName(subDbName) { if (LOG_WOULD_LOG(event)) { @@ -58,4 +61,18 @@ AttributePopulator::handleExisting(uint32_t lid, const document::Document &doc) _writer.put(serialNum, doc, lid, true, std::shared_ptr<IDestructorCallback>()); } +void +AttributePopulator::done() +{ + auto mgr = _writer.getAttributeManager(); + auto flushTargets = mgr->getFlushTargets(); + for (const auto &flushTarget : flushTargets) { + assert(flushTarget->getFlushedSerialNum() < _configSerialNum); + auto task = flushTarget->initFlush(_configSerialNum); + assert(task); + task->run(); + assert(flushTarget->getFlushedSerialNum() == _configSerialNum); + } +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h index b6c53af6bac..e918499fb5b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.h @@ -16,6 +16,7 @@ private: AttributeWriter _writer; search::SerialNum _initSerialNum; search::SerialNum _currSerialNum; + search::SerialNum _configSerialNum; vespalib::string _subDbName; search::SerialNum nextSerialNum(); @@ -27,13 +28,15 @@ public: AttributePopulator(const proton::IAttributeManager::SP &mgr, search::SerialNum initSerialNum, - const vespalib::string &subDbName); + const vespalib::string &subDbName, + search::SerialNum configSerialNum); ~AttributePopulator(); const IAttributeWriter &getWriter() const { return _writer; } // Implements IReprocessingReader - virtual void handleExisting(uint32_t lid, const document::Document &doc); + virtual void handleExisting(uint32_t lid, const document::Document &doc) override; + virtual void done() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 6ddd018c179..fa4f9258e36 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -27,9 +27,40 @@ using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using search::common::FileHeaderContext; using search::index::Schema; +using search::attribute::BasicType; namespace proton { +namespace { + +bool matchingTypes(const AttributeVector::SP &av, const search::attribute::Config &newConfig) { + if (av) { + const auto &oldConfig = av->getConfig(); + if ((oldConfig.basicType() != newConfig.basicType()) || + (oldConfig.collectionType() != newConfig.collectionType())) { + return false; + } + if (newConfig.basicType().type() == BasicType::TENSOR) { + if (oldConfig.tensorType() != newConfig.tensorType()) { + return false; + } + } + if (newConfig.basicType().type() == BasicType::PREDICATE) { + using Params = search::attribute::PersistentPredicateParams; + const Params &oldParams = oldConfig.predicateParams(); + const Params &newParams = newConfig.predicateParams(); + if (!(oldParams == newParams)) { + return false; + } + } + return true; + } else { + return false; + } +} + +} + AttributeVector::SP AttributeManager::internalAddAttribute(const vespalib::string &name, const Config &cfg, @@ -86,7 +117,7 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, { for (const auto &aspec : newSpec.getAttributes()) { AttributeVector::SP av = currMgr.findAttribute(aspec.getName()); - if (av.get() != NULL) { // transfer attribute + if (matchingTypes(av, aspec.getConfig())) { // transfer attribute LOG(debug, "Transferring attribute vector '%s' with %u docs and serial number %lu from current manager", av->getName().c_str(), av->getNumDocs(), av->getStatus().getLastSyncToken()); addAttribute(av); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index d6540331345..2610f243612 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -6,9 +6,30 @@ #include <vespa/vespalib/util/exceptions.h> using search::AttributeGuard; +using searchcorespi::IFlushTarget; namespace proton { +namespace { + +const vespalib::string FLUSH_TARGET_NAME_PREFIX("attribute."); + +bool isAttributeFlushTarget(const IFlushTarget::SP &flushTarget) { + const vespalib::string &targetName = flushTarget->getName(); + if ((flushTarget->getType() != IFlushTarget::Type::SYNC) || + (flushTarget->getComponent() != IFlushTarget::Component::ATTRIBUTE)) { + return false; + } + return (targetName.substr(0, FLUSH_TARGET_NAME_PREFIX.size()) == FLUSH_TARGET_NAME_PREFIX); +} + +vespalib::string attributeFlushTargetAttributeName(const IFlushTarget::SP &flushTarget) { + const vespalib::string &targetName = flushTarget->getName(); + return targetName.substr(FLUSH_TARGET_NAME_PREFIX.size()); +} + +} + bool FilterAttributeManager::acceptAttribute(const vespalib::string &name) const { @@ -43,10 +64,22 @@ IAttributeManager::SP FilterAttributeManager::create(const AttributeCollectionSpec &) const { throw vespalib::IllegalArgumentException("Not implemented"); } + std::vector<searchcorespi::IFlushTarget::SP> FilterAttributeManager::getFlushTargets() const { - throw vespalib::IllegalArgumentException("Not implemented"); + std::vector<searchcorespi::IFlushTarget::SP> completeList = _mgr->getFlushTargets(); + std::vector<searchcorespi::IFlushTarget::SP> list; + list.reserve(completeList.size()); + for (const auto &flushTarget : completeList) { + if (isAttributeFlushTarget(flushTarget)) { + if (acceptAttribute(attributeFlushTargetAttributeName(flushTarget))) { + list.push_back(flushTarget); + } + } + } + return list; } + search::SerialNum FilterAttributeManager::getOldestFlushedSerialNumber() const { throw vespalib::IllegalArgumentException("Not implemented"); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 5fc4dceea7a..209d2cf07f3 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -235,7 +235,7 @@ DocumentMetaStore::onInitSave() { GenerationHandler::Guard guard(getGuard()); return std::make_unique<DocumentMetaStoreSaver> - (std::move(guard), createSaveTargetConfig(), + (std::move(guard), createAttributeHeader(), _gidToLidMap.getFrozenView().begin(), _metaDataStore); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 789706503cf..200535c7ae5 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -12,7 +12,6 @@ #include "raw_document_meta_data.h" #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/common/subdbtype.h> -#include <vespa/searchlib/attribute/iattributesavetarget.h> #include <vespa/searchlib/common/rcuvector.h> #include <vespa/searchlib/attribute/singlesmallnumericattribute.h> #include <vespa/searchlib/queryeval/blueprint.h> diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp index ce7869e848e..4ca5d07d090 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp @@ -3,6 +3,7 @@ #include "documentmetastoresaver.h" #include <vespa/searchlib/util/bufferwriter.h> #include "document_meta_store_versions.h" +#include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; using search::IAttributeSaveTarget; @@ -67,15 +68,15 @@ public: DocumentMetaStoreSaver:: DocumentMetaStoreSaver(vespalib::GenerationHandler::Guard &&guard, - const search::IAttributeSaveTarget::Config &cfg, - const GidIterator &gidIterator, - const MetaDataStore &metaDataStore) - : AttributeSaver(std::move(guard), cfg), + const search::attribute::AttributeHeader &header, + const GidIterator &gidIterator, + const MetaDataStore &metaDataStore) + : AttributeSaver(std::move(guard), header), _gidIterator(gidIterator), _metaDataStore(metaDataStore), _writeDocSize(true) { - if (cfg.getVersion() == documentmetastore::NO_DOCUMENT_SIZE_TRACKING_VERSION) { + if (header.getVersion() == documentmetastore::NO_DOCUMENT_SIZE_TRACKING_VERSION) { _writeDocSize = false; } } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h index ddec7032faa..9a8aba5387f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h @@ -35,9 +35,9 @@ private: virtual bool onSave(search::IAttributeSaveTarget &saveTarget) override; public: DocumentMetaStoreSaver(vespalib::GenerationHandler::Guard &&guard, - const search::IAttributeSaveTarget::Config &cfg, - const GidIterator &gidIterator, - const MetaDataStore &metaDataStore); + const search::attribute::AttributeHeader &header, + const GidIterator &gidIterator, + const MetaDataStore &metaDataStore); virtual ~DocumentMetaStoreSaver(); }; diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index d1cb05292c1..64587f03883 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -36,7 +36,8 @@ bool fastPartialUpdateAttribute(const schema::DataType &attrType) { FilterAttributeManager::AttributeSet getAttributeSetToPopulate(const ARIConfig &newCfg, - const ARIConfig &oldCfg) + const ARIConfig &oldCfg, + search::SerialNum serialNum) { FilterAttributeManager::AttributeSet attrsToPopulate; std::vector<AttributeGuard> attrList; @@ -45,7 +46,8 @@ getAttributeSetToPopulate(const ARIConfig &newCfg, const vespalib::string &name = guard->getName(); bool inOldAttrMgr = oldCfg.getAttrMgr()->getAttribute(name)->valid(); bool inOldSchema = oldCfg.getInspector()->hasField(name); - bool populateAttribute = !inOldAttrMgr && inOldSchema; + search::SerialNum flushedSerialNum = newCfg.getAttrMgr()->getFlushedSerialNum(name); + bool populateAttribute = !inOldAttrMgr && inOldSchema && (flushedSerialNum < serialNum); LOG(debug, "getAttributeSetToPopulate(): name='%s', inOldAttrMgr=%s, inOldSchema=%s, populate=%s", name.c_str(), toStr(inOldAttrMgr), toStr(inOldSchema), toStr(populateAttribute)); if (populateAttribute) { @@ -58,15 +60,16 @@ getAttributeSetToPopulate(const ARIConfig &newCfg, IReprocessingReader::SP getAttributesToPopulate(const ARIConfig &newCfg, const ARIConfig &oldCfg, - const vespalib::string &subDbName) + const vespalib::string &subDbName, + search::SerialNum serialNum) { FilterAttributeManager::AttributeSet attrsToPopulate = - getAttributeSetToPopulate(newCfg, oldCfg); + getAttributeSetToPopulate(newCfg, oldCfg, serialNum); if (!attrsToPopulate.empty()) { return IReprocessingReader::SP(new AttributePopulator (IAttributeManager::SP(new FilterAttributeManager (attrsToPopulate, newCfg.getAttrMgr())), - ATTRIBUTE_INIT_SERIAL, subDbName)); + ATTRIBUTE_INIT_SERIAL, subDbName, serialNum)); } return IReprocessingReader::SP(); } @@ -119,8 +122,9 @@ getFieldsToPopulate(const ARIConfig &newCfg, AttributeReprocessingInitializer:: AttributeReprocessingInitializer(const Config &newCfg, const Config &oldCfg, - const vespalib::string &subDbName) - : _attrsToPopulate(getAttributesToPopulate(newCfg, oldCfg, subDbName)), + const vespalib::string &subDbName, + search::SerialNum serialNum) + : _attrsToPopulate(getAttributesToPopulate(newCfg, oldCfg, subDbName, serialNum)), _fieldsToPopulate(getFieldsToPopulate(newCfg, oldCfg, subDbName)) { } diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h index f1837a6dbcb..bb2b4de4897 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h @@ -46,7 +46,8 @@ private: public: AttributeReprocessingInitializer(const Config &newCfg, const Config &oldCfg, - const vespalib::string &subDbName); + const vespalib::string &subDbName, + search::SerialNum serialNum); // Implements IReprocessingInitializer virtual bool hasReprocessors() const; diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp index 4da9bdf76d9..ff538ea8c68 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.cpp @@ -45,4 +45,12 @@ DocumentReprocessingHandler::visit(uint32_t lid) (void) lid; } +void +DocumentReprocessingHandler::done() +{ + for (const auto &reader : _readers) { + reader->done(); + } +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h index 32e472beaa0..f950e3b61a9 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h @@ -70,6 +70,7 @@ public: virtual void visit(uint32_t lid); + void done(); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h index e265befe257..f8c800f6e2e 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/i_reprocessing_reader.h @@ -20,6 +20,7 @@ struct IReprocessingReader * Handle the given existing document. */ virtual void handleExisting(uint32_t lid, const document::Document &doc) = 0; + virtual void done() = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp index 9084c3d8e63..59b3966454a 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp @@ -53,6 +53,7 @@ ReprocessDocumentsTask::run() *this, *_docTypeRepo); } + _handler.done(); ts = fastos::ClockSystem::now(); int64_t elapsedTime = ts.ms() - _startTime; EventLogger::reprocessDocumentsComplete(_subDbName, diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp index 2698ffdda69..5f77400d7fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp @@ -64,7 +64,7 @@ FastAccessDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, IDocumentTypeInspector::SP(new DocumentTypeInspector(*newDocType))), ARIConfig(oldView->getAttributeWriter()->getAttributeManager(), *oldConfig.getSchemaSP(), IDocumentTypeInspector::SP(new DocumentTypeInspector(*oldDocType))), - _subDbName)); + _subDbName, attrSpec.getCurrentSerialNum())); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 202efae5397..c6b3a86239c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -164,7 +164,8 @@ createAttributeReprocessingInitializer(const DocumentDBConfig &newConfig, const IAttributeManager::SP &newAttrMgr, const DocumentDBConfig &oldConfig, const IAttributeManager::SP &oldAttrMgr, - const vespalib::string &subDbName) + const vespalib::string &subDbName, + search::SerialNum serialNum) { const document::DocumentType *newDocType = newConfig.getDocumentType(); const document::DocumentType *oldDocType = oldConfig.getDocumentType(); @@ -175,7 +176,7 @@ createAttributeReprocessingInitializer(const DocumentDBConfig &newConfig, IDocumentTypeInspector::SP(new DocumentTypeInspector(*newDocType))), ARIConfig(oldAttrMgr, *oldConfig.getSchemaSP(), IDocumentTypeInspector::SP(new DocumentTypeInspector(*oldDocType))), - subDbName)); + subDbName, serialNum)); } } @@ -216,7 +217,7 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, attrWriter = newAttrWriter; shouldFeedViewChange = true; initializer.reset(createAttributeReprocessingInitializer(newConfig, newAttrMgr, - oldConfig, oldAttrMgr, _subDbName).release()); + oldConfig, oldAttrMgr, _subDbName, attrSpec.getCurrentSerialNum()).release()); } ISummaryManager::ISummarySetup::SP sumSetup = diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index d38c1714a65..b1786a36597 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -217,6 +217,8 @@ private: void testCreateSerialNum(void); + void testPredicateHeaderTags(); + template <typename VectorType, typename BufferType> void testCompactLidSpace(const Config &config, @@ -1994,6 +1996,21 @@ AttributeTest::testCreateSerialNum() EXPECT_EQUAL(42u, attr2->getCreateSerialNum()); } +void +AttributeTest::testPredicateHeaderTags() +{ + Config cfg(BasicType::PREDICATE); + AttributePtr attr = createAttribute("predicate", cfg); + attr->addReservedDoc(); + EXPECT_TRUE(attr->save()); + auto df = search::FileUtil::openFile(baseFileName("predicate.dat")); + vespalib::FileHeader datHeader; + datHeader.readFile(*df); + EXPECT_TRUE(datHeader.hasTag("predicate.arity")); + EXPECT_TRUE(datHeader.hasTag("predicate.lower_bound")); + EXPECT_TRUE(datHeader.hasTag("predicate.upper_bound")); + EXPECT_EQUAL(8u, datHeader.getTag("predicate.arity").asInteger()); +} template <typename VectorType, typename BufferType> void @@ -2341,6 +2358,7 @@ int AttributeTest::Main() testNullProtection(); testGeneration(); testCreateSerialNum(); + testPredicateHeaderTags(); TEST_DO(testCompactLidSpace()); TEST_DO(requireThatAddressSpaceUsageIsReported()); testReaderDuringLastUpdate(); diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index 57d50ea9537..afba4287424 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -7,6 +7,7 @@ LOG_SETUP("attributefilewriter_test"); #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/attribute/attributefilewriter.h> #include <vespa/searchlib/attribute/attributefilebufferwriter.h> +#include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/util/fileutil.h> #include <vespa/searchlib/util/rand48.h> #include <vespa/searchlib/common/tunefileinfo.h> @@ -29,18 +30,18 @@ void removeTestFile() { FastOS_File::Delete(testFileName.c_str()); } struct Fixture { TuneFileAttributes _tuneFileAttributes; DummyFileHeaderContext _fileHeaderContext; - IAttributeSaveTarget::Config _cfg; + attribute::AttributeHeader _header; const vespalib::string _desc; AttributeFileWriter _writer; Fixture() : _tuneFileAttributes(), _fileHeaderContext(), - _cfg(), + _header(), _desc("Attribute file sample description"), _writer(_tuneFileAttributes, _fileHeaderContext, - _cfg, + _header, _desc) { removeTestFile(); diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index e167dc38f22..e06514e3fc0 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(searchlib_attribute OBJECT address_space_usage.cpp attribute.cpp attribute_blueprint_factory.cpp + attribute_header.cpp attribute_weighted_set_blueprint.cpp attributecontext.cpp attributefactory.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp new file mode 100644 index 00000000000..84f0720b140 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -0,0 +1,154 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_header.h" +#include <vespa/vespalib/data/fileheader.h> + +namespace search { +namespace attribute { + +namespace { + +const vespalib::string versionTag = "version"; +const vespalib::string dataTypeTag = "datatype"; +const vespalib::string collectionTypeTag = "collectiontype"; +const vespalib::string createSerialNumTag = "createSerialNum"; +const vespalib::string tensorTypeTag = "tensortype"; +const vespalib::string predicateArityTag = "predicate.arity"; +const vespalib::string predicateLowerBoundTag = "predicate.lower_bound"; +const vespalib::string predicateUpperBoundTag = "predicate.upper_bound"; + +} + +AttributeHeader::AttributeHeader() + : _fileName(""), + _basicType(attribute::BasicType::Type::NONE), + _collectionType(attribute::CollectionType::Type::SINGLE), + _tensorType(vespalib::eval::ValueType::error_type()), + _enumerated(false), + _predicateParamsSet(false), + _predicateParams(), + _numDocs(0), + _fixedWidth(0), + _uniqueValueCount(0), + _totalValueCount(0), + _createSerialNum(0u), + _version(0) +{ +} + +AttributeHeader::AttributeHeader(const vespalib::string &fileName, + attribute::BasicType basicType, + attribute::CollectionType collectionType, + const vespalib::eval::ValueType &tensorType, + bool enumerated, + const attribute::PersistentPredicateParams &predicateParams, + uint32_t numDocs, + uint32_t fixedWidth, + uint64_t uniqueValueCount, + uint64_t totalValueCount, + uint64_t createSerialNum, + uint32_t version) + : _fileName(fileName), + _basicType(basicType), + _collectionType(collectionType), + _tensorType(tensorType), + _enumerated(enumerated), + _predicateParamsSet(false), + _predicateParams(predicateParams), + _numDocs(numDocs), + _fixedWidth(fixedWidth), + _uniqueValueCount(uniqueValueCount), + _totalValueCount(totalValueCount), + _createSerialNum(createSerialNum), + _version(version) +{ +} + +AttributeHeader::~AttributeHeader() +{ +} + +void +AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header) +{ + if (header.hasTag(createSerialNumTag)) { + _createSerialNum = header.getTag(createSerialNumTag).asInteger(); + } + if (header.hasTag(dataTypeTag)) { + _basicType = BasicType(header.getTag(dataTypeTag).asString()); + } + if (header.hasTag(collectionTypeTag)) { + _collectionType = CollectionType(header.getTag(collectionTypeTag).asString()); + } + if (_basicType.type() == BasicType::Type::TENSOR) { + assert(header.hasTag(tensorTypeTag)); + _tensorType = vespalib::eval::ValueType::from_spec(header.getTag(tensorTypeTag).asString()); + } + if (_basicType.type() == BasicType::Type::PREDICATE) { + if (header.hasTag(predicateArityTag)) { + assert(header.hasTag(predicateLowerBoundTag)); + assert(header.hasTag(predicateUpperBoundTag)); + _predicateParamsSet = true; + _predicateParams.setArity(header.getTag(predicateArityTag).asInteger()); + _predicateParams.setBounds(header.getTag(predicateLowerBoundTag).asInteger(), header.getTag(predicateUpperBoundTag).asInteger()); + } else { + assert(!header.hasTag(predicateLowerBoundTag)); + assert(!header.hasTag(predicateUpperBoundTag)); + } + } + if (header.hasTag(versionTag)) { + _version = header.getTag(versionTag).asInteger(); + } +} + +AttributeHeader +AttributeHeader::extractTags(const vespalib::GenericHeader &header) +{ + AttributeHeader result; + result.internalExtractTags(header); + return result; +} + +void +AttributeHeader::addTags(vespalib::GenericHeader &header) const +{ + using Tag = vespalib::GenericHeader::Tag; + header.putTag(Tag(dataTypeTag, _basicType.asString())); + header.putTag(Tag(collectionTypeTag, _collectionType.asString())); + header.putTag(Tag("uniqueValueCount", _uniqueValueCount)); + header.putTag(Tag("totalValueCount", _totalValueCount)); + header.putTag(Tag("docIdLimit", _numDocs)); + header.putTag(Tag("frozen", 0)); + header.putTag(Tag("fileBitSize", 0)); + header.putTag(Tag(versionTag, _version)); + if (_enumerated) { + header.putTag(Tag("enumerated", 1)); + } + if (_createSerialNum != 0u) { + header.putTag(Tag(createSerialNumTag, _createSerialNum)); + } + if (_basicType.type() == attribute::BasicType::Type::TENSOR) { + header.putTag(Tag(tensorTypeTag, _tensorType.to_spec()));; + } + if (_basicType.type() == attribute::BasicType::Type::PREDICATE) { + const auto & params = _predicateParams; + header.putTag(Tag(predicateArityTag, params.arity())); + header.putTag(Tag(predicateLowerBoundTag, params.lower_bound())); + header.putTag(Tag(predicateUpperBoundTag, params.upper_bound())); + } +} + +bool +AttributeHeader::hasMultiValue() const +{ + return _collectionType.isMultiValue(); +} + +bool +AttributeHeader::hasWeightedSetType() const +{ + return _collectionType.isWeightedSet(); +} + +} // namespace search::attribute +} // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.h b/searchlib/src/vespa/searchlib/attribute/attribute_header.h new file mode 100644 index 00000000000..7e9ef78dafb --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.h @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/searchcommon/attribute/basictype.h> +#include <vespa/searchcommon/attribute/collectiontype.h> +#include <vespa/searchcommon/attribute/predicate_params.h> +#include <vespa/eval/eval/value_type.h> + +namespace vespalib { class GenericHeader; } + +namespace search { +namespace attribute { + +/** + * Attribute header class used by attribute savers and attribute initializer + * to convert to/from generic header tags. + **/ +class AttributeHeader { +private: + vespalib::string _fileName; + BasicType _basicType; + CollectionType _collectionType; + vespalib::eval::ValueType _tensorType; + bool _enumerated; + bool _predicateParamsSet; + PersistentPredicateParams _predicateParams; + uint32_t _numDocs; + uint32_t _fixedWidth; + uint64_t _uniqueValueCount; + uint64_t _totalValueCount; + uint64_t _createSerialNum; + uint32_t _version; + + void internalExtractTags(const vespalib::GenericHeader &header); +public: + AttributeHeader(); + AttributeHeader(const vespalib::string &fileName, + BasicType basicType, + CollectionType collectionType, + const vespalib::eval::ValueType &tensorType, + bool enumerated, + const PersistentPredicateParams &predicateParams, + uint32_t numDocs, + uint32_t fixedWidth, + uint64_t uniqueValueCount, + uint64_t totalValueCount, + uint64_t createSerialNum, + uint32_t version); + ~AttributeHeader(); + + const vespalib::string & getFileName() const { return _fileName; } + const BasicType & getBasicType() const { return _basicType; } + const CollectionType &getCollectionType() const { return _collectionType; } + const vespalib::eval::ValueType &getTensorType() const { return _tensorType; } + bool hasMultiValue() const; + bool hasWeightedSetType() const; + uint32_t getNumDocs() const { return _numDocs; } + size_t getFixedWidth() const { return _fixedWidth; } + uint64_t getUniqueValueCount(void) const { return _uniqueValueCount; } + uint64_t getTotalValueCount(void) const { return _totalValueCount; } + bool getEnumerated(void) const { return _enumerated; } + uint64_t getCreateSerialNum(void) const { return _createSerialNum; } + uint32_t getVersion() const { return _version; } + const PersistentPredicateParams &getPredicateParams() const { return _predicateParams; } + bool getPredicateParamsSet() const { return _predicateParamsSet; } + static AttributeHeader extractTags(const vespalib::GenericHeader &header); + void addTags(vespalib::GenericHeader &header) const; +}; + +} // namespace search::attribute +} // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp index c9a846d47df..ce77c3b8722 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp @@ -23,13 +23,13 @@ AttributeFileSaveTarget:: AttributeFileSaveTarget(const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext) : IAttributeSaveTarget(), - _datWriter(tuneFileAttributes, fileHeaderContext, _cfg, + _datWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector data file"), - _idxWriter(tuneFileAttributes, fileHeaderContext, _cfg, + _idxWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector idx file"), - _weightWriter(tuneFileAttributes, fileHeaderContext, _cfg, + _weightWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector weight file"), - _udatWriter(tuneFileAttributes, fileHeaderContext, _cfg, + _udatWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector unique data file") { } @@ -40,23 +40,23 @@ AttributeFileSaveTarget::~AttributeFileSaveTarget() { bool AttributeFileSaveTarget::setup() { - const vespalib::string & baseFileName = _cfg.getFileName(); + const vespalib::string & baseFileName = _header.getFileName(); vespalib::string datFileName(baseFileName + ".dat"); if (!_datWriter.open(datFileName)) { return false; } - if (_cfg.getEnumerated()) { + if (_header.getEnumerated()) { vespalib::string udatFileName(baseFileName + ".udat"); if (!_udatWriter.open(udatFileName)) { return false; } } - if (_cfg.hasMultiValue()) { + if (_header.hasMultiValue()) { vespalib::string idxFileName(baseFileName + ".idx"); if (!_idxWriter.open(idxFileName)) { return false; } - if (_cfg.hasWeightedSetType()) { + if (_header.hasWeightedSetType()) { vespalib::string weightFileName(baseFileName + ".weight"); if (!_weightWriter.open(weightFileName)) { return false; diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp index 2abd74d603f..e3e9fd01a2c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/common/tunefileinfo.h> +#include "attribute_header.h" #include <vespa/fastos/file.h> #include <vespa/log/log.h> @@ -96,12 +97,12 @@ FileBackedBufferWriter::onFlush(size_t nowLen) { AttributeFileWriter:: AttributeFileWriter(const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const vespalib::string &desc) : _file(new FastOS_File()), _tuneFileAttributes(tuneFileAttributes), _fileHeaderContext(fileHeaderContext), - _cfg(cfg), + _header(header), _desc(desc), _fileBitSize(0) { } @@ -145,26 +146,8 @@ AttributeFileWriter::writeHeader() void AttributeFileWriter::addTags(vespalib::GenericHeader &header) { - typedef vespalib::GenericHeader::Tag Tag; - header.putTag(Tag("datatype", _cfg.getBasicType())); - header.putTag(Tag("collectiontype", _cfg.getCollectionType())); - header.putTag(Tag("uniqueValueCount", _cfg.getUniqueValueCount())); - header.putTag(Tag("totalValueCount", _cfg.getTotalValueCount())); - header.putTag(Tag("docIdLimit", _cfg.getNumDocs())); - header.putTag(Tag("frozen", 0)); - header.putTag(Tag("fileBitSize", 0)); - header.putTag(Tag("version", _cfg.getVersion())); - if (_cfg.getEnumerated()) { - header.putTag(Tag("enumerated", 1)); - } - uint64_t createSerialNum = _cfg.getCreateSerialNum(); - if (createSerialNum != 0u) { - header.putTag(Tag("createSerialNum", createSerialNum)); - } - const vespalib::string &tensorType = _cfg.getTensorType(); - if (!tensorType.empty()) { - header.putTag(Tag("tensortype", tensorType));; - } + _header.addTags(header); + using Tag = vespalib::GenericHeader::Tag; header.putTag(Tag("desc", _desc)); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h index 5693b0be9d2..b431f7b6ae6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h @@ -2,8 +2,8 @@ #pragma once -#include "iattributesavetarget.h" #include "iattributefilewriter.h" +#include <vespa/vespalib/stllike/string.h> class FastOS_FileInterface; @@ -12,6 +12,7 @@ namespace vespalib { class GenericHeader; } namespace search { namespace common { class FileHeaderContext; } +namespace attribute { class AttributeHeader; } class TuneFileAttributes; @@ -24,7 +25,7 @@ class AttributeFileWriter : public IAttributeFileWriter std::unique_ptr<FastOS_FileInterface> _file; const TuneFileAttributes &_tuneFileAttributes; const search::common::FileHeaderContext &_fileHeaderContext; - const IAttributeSaveTarget::Config &_cfg; + const attribute::AttributeHeader &_header; vespalib::string _desc; uint64_t _fileBitSize; @@ -34,7 +35,7 @@ class AttributeFileWriter : public IAttributeFileWriter public: AttributeFileWriter(const TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const vespalib::string &desc); ~AttributeFileWriter(); virtual Buffer allocBuf(size_t size) override; diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp index cfc4061d4d5..41166c8b76b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp @@ -59,17 +59,17 @@ writeToFile(const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext) { AttributeFileSaveTarget saveTarget(tuneFileAttributes, fileHeaderContext); - saveTarget.setConfig(_cfg); + saveTarget.setHeader(_header); if (!saveTarget.setup()) { return false; } _datWriter.writeTo(saveTarget.datWriter()); - if (_cfg.getEnumerated()) { + if (_header.getEnumerated()) { _udatWriter.writeTo(saveTarget.udatWriter()); } - if (_cfg.hasMultiValue()) { + if (_header.hasMultiValue()) { _idxWriter.writeTo(saveTarget.idxWriter()); - if (_cfg.hasWeightedSetType()) { + if (_header.hasWeightedSetType()) { _weightWriter.writeTo(saveTarget.weightWriter()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp index 4d3649712d3..f00a14127aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributesaver.cpp @@ -2,6 +2,7 @@ #include <vespa/fastos/fastos.h> #include "attributesaver.h" +#include "iattributesavetarget.h" using vespalib::GenerationHandler; @@ -10,9 +11,9 @@ namespace search { AttributeSaver::AttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg) + const attribute::AttributeHeader &header) : _guard(std::move(guard)), - _cfg(cfg) + _header(header) { } @@ -25,7 +26,7 @@ AttributeSaver::~AttributeSaver() bool AttributeSaver::save(IAttributeSaveTarget &saveTarget) { - saveTarget.setConfig(_cfg); + saveTarget.setHeader(_header); if (!saveTarget.setup()) { return false; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributesaver.h b/searchlib/src/vespa/searchlib/attribute/attributesaver.h index 307239e113b..c9787cd923f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/attributesaver.h @@ -3,11 +3,13 @@ #pragma once #include <vespa/vespalib/util/generationhandler.h> -#include "iattributesavetarget.h" +#include "attribute_header.h" namespace search { +class IAttributeSaveTarget; + /* * Abstract class used to hold data outside attribute vector needed * during a save operation, e.g. copy of data structure without @@ -18,11 +20,11 @@ class AttributeSaver { private: vespalib::GenerationHandler::Guard _guard; - IAttributeSaveTarget::Config _cfg; + attribute::AttributeHeader _header; protected: AttributeSaver(vespalib::GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg); + const attribute::AttributeHeader &header); virtual bool onSave(IAttributeSaveTarget &saveTarget) = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index f0a0b72a3e6..6ba598f1f3e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -456,7 +456,7 @@ AttributeVector::save(IAttributeSaveTarget &saveTarget) return saver->save(saveTarget); } // New style save not available, use old style save - saveTarget.setConfig(createSaveTargetConfig()); + saveTarget.setHeader(createAttributeHeader()); if (!saveTarget.setup()) { return false; } @@ -465,17 +465,16 @@ AttributeVector::save(IAttributeSaveTarget &saveTarget) return true; } -IAttributeSaveTarget::Config -AttributeVector::createSaveTargetConfig() const { - return IAttributeSaveTarget::Config(getBaseFileName(), - getConfig().basicType().asString(), - getConfig().collectionType().asString(), +attribute::AttributeHeader +AttributeVector::createAttributeHeader() const { + return attribute::AttributeHeader(getBaseFileName(), + getConfig().basicType(), + getConfig().collectionType(), getConfig().basicType().type() == BasicType::Type::TENSOR - ? getConfig().tensorType().to_spec() - : "", - hasMultiValue(), - hasWeightedSetType(), + ? getConfig().tensorType() + : vespalib::eval::ValueType::error_type(), getEnumeratedSave(), + getConfig().predicateParams(), getCommittedDocIdLimit(), getFixedWidth(), getUniqueValueCount(), diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 64a9cb0adf8..f313c77f581 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -3,7 +3,6 @@ #pragma once #include "address_space_usage.h" -#include "iattributesavetarget.h" #include "changevector.h" #include <vespa/fastlib/text/normwordfolder.h> #include <vespa/searchcommon/attribute/config.h> @@ -46,6 +45,7 @@ namespace search { class AttributeWriteGuard; class AttributeSaver; class EnumStoreBase; + class IAttributeSaveTarget; class IDocumentWeightAttribute; class QueryTermSimple; class QueryTermBase; @@ -55,6 +55,7 @@ namespace search { } namespace attribute { + class AttributeHeader; class IPostingListSearchContext; class IPostingListAttributeBase; class Interlock; @@ -69,6 +70,7 @@ namespace search { namespace search { + using search::attribute::WeightedType; using search::attribute::Status; using document::ArithmeticValueUpdate; @@ -439,7 +441,7 @@ public: /** Saves this attribute vector using the given saveTarget **/ bool save(IAttributeSaveTarget & saveTarget); - IAttributeSaveTarget::Config createSaveTargetConfig() const; + attribute::AttributeHeader createAttributeHeader() const; /** Returns whether this attribute has load data files on disk **/ bool hasLoadData() const; diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp index b1cd962b190..b59b3aa81d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp @@ -7,6 +7,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/attrvector.h> #include <vespa/searchlib/util/filekit.h> +#include "iattributesavetarget.h" LOG_SETUP(".attrvector"); diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 8c911a78d43..9e3c6156dbf 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -65,15 +65,17 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) cType.removeIfZero(cfg.removeifzero); cType.createIfNonExistant(cfg.createifnonexistent); Config retval(bType, cType); + PredicateParams predicateParams; retval.setFastSearch(cfg.fastsearch); retval.setHuge(cfg.huge); retval.setEnableBitVectors(cfg.enablebitvectors); retval.setEnableOnlyBitVector(cfg.enableonlybitvector); retval.setIsFilter(cfg.enableonlybitvector); retval.setFastAccess(cfg.fastaccess); - retval.setArity(cfg.arity); - retval.setBounds(cfg.lowerbound, cfg.upperbound); - retval.setDensePostingListThreshold(cfg.densepostinglistthreshold); + predicateParams.setArity(cfg.arity); + predicateParams.setBounds(cfg.lowerbound, cfg.upperbound); + predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold); + retval.setPredicateParams(predicateParams); if (retval.basicType().type() == BasicType::Type::TENSOR) { if (!cfg.tensortype.empty()) { retval.setTensorType(ValueType::from_spec(cfg.tensortype)); diff --git a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp index ab9931a7fbf..e7221c0c401 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp +++ b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.cpp @@ -7,9 +7,6 @@ LOG_SETUP(".searchlib.attribute.iattributesavetarget"); namespace search { -IAttributeSaveTarget::Config::~Config() { -} - IAttributeSaveTarget::~IAttributeSaveTarget() { } diff --git a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h index d0ab72983a7..40d02a4b5c9 100644 --- a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h +++ b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/databuffer.h> #include <stdint.h> #include "iattributefilewriter.h" +#include "attribute_header.h" namespace search { @@ -14,132 +15,14 @@ namespace search { **/ class IAttributeSaveTarget { public: - /** - * Config class used by actual saveTarget implementations. - **/ - class Config { - private: - vespalib::string _fileName; - vespalib::string _basicType; - vespalib::string _collectionType; - vespalib::string _tensorType; - bool _hasMultiValue; - bool _hasWeightedSetType; - bool _enumerated; - uint32_t _numDocs; - uint32_t _fixedWidth; - uint64_t _uniqueValueCount; - uint64_t _totalValueCount; - uint64_t _createSerialNum; - uint32_t _version; - public: - Config() - : _fileName(""), - _basicType(""), - _collectionType(""), - _hasMultiValue(false), - _hasWeightedSetType(false), - _enumerated(false), - _numDocs(0), - _fixedWidth(0), - _uniqueValueCount(0), - _totalValueCount(0), - _createSerialNum(0u), - _version(0) - { - } - - Config(const vespalib::string &fileName, - const vespalib::string &basicType, - const vespalib::string &collectionType, - const vespalib::string &tensorType, - bool multiValue, bool weightedSetType, - bool enumerated, - uint32_t numDocs, - uint32_t fixedWidth, - uint64_t uniqueValueCount, - uint64_t totalValueCount, - uint64_t createSerialNum, - uint32_t version - ) - : _fileName(fileName), - _basicType(basicType), - _collectionType(collectionType), - _tensorType(tensorType), - _hasMultiValue(multiValue), - _hasWeightedSetType(weightedSetType), - _enumerated(enumerated), - _numDocs(numDocs), - _fixedWidth(fixedWidth), - _uniqueValueCount(uniqueValueCount), - _totalValueCount(totalValueCount), - _createSerialNum(createSerialNum), - _version(version) - { - } - ~Config(); - - const vespalib::string & getFileName() const { return _fileName; } - - const vespalib::string & - getBasicType() const - { - return _basicType; - } - - const vespalib::string & - getCollectionType() const - { - return _collectionType; - } - - const vespalib::string &getTensorType() const { - return _tensorType; - } - - bool hasMultiValue() const { return _hasMultiValue; } - bool hasWeightedSetType() const { return _hasWeightedSetType; } - uint32_t getNumDocs() const { return _numDocs; } - size_t getFixedWidth() const { return _fixedWidth; } - - uint64_t - getUniqueValueCount(void) const - { - return _uniqueValueCount; - } - - uint64_t - getTotalValueCount(void) const - { - return _totalValueCount; - } - - bool - getEnumerated(void) const - { - return _enumerated; - } - - uint64_t - getCreateSerialNum(void) const - { - return _createSerialNum; - } - - uint32_t getVersion() const { return _version; } - }; using Buffer = IAttributeFileWriter::Buffer; protected: - Config _cfg; + attribute::AttributeHeader _header; public: - IAttributeSaveTarget() : _cfg() {} - void setConfig(const Config & cfg) { _cfg = cfg; } + IAttributeSaveTarget() : _header() {} + void setHeader(const attribute::AttributeHeader & header) { _header = header; } - bool - getEnumerated(void) const - { - return _cfg.getEnumerated(); - } + bool getEnumerated(void) const { return _header.getEnumerated(); } /** * Setups this saveTarget before any data is written. Returns true diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index 712b51f5ea7..ddbca2f0fc8 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -203,7 +203,7 @@ MultiValueEnumAttribute<B, M>::onInitSave() vespalib::GenerationHandler::Guard guard(this->getGenerationHandler(). takeGuard()); return std::make_unique<MultiValueEnumAttributeSaver<WeightedIndex>> - (std::move(guard), this->createSaveTargetConfig(), this->_mvMapping, + (std::move(guard), this->createAttributeHeader(), this->_mvMapping, this->_enumStore); } diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp index 8b876e2833e..76917ee883f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp @@ -73,10 +73,10 @@ public: template <typename MultiValueT> MultiValueEnumAttributeSaver<MultiValueT>:: MultiValueEnumAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const MultiValueMapping &mvMapping, const EnumStoreBase &enumStore) - : Parent(std::move(guard), cfg, mvMapping), + : Parent(std::move(guard), header, mvMapping), _mvMapping(mvMapping), _enumSaver(enumStore, true) { diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h index fc8f3576165..eab8067b024 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h @@ -28,7 +28,7 @@ class MultiValueEnumAttributeSaver : public MultiValueAttributeSaver public: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; MultiValueEnumAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const MultiValueMapping &mvMapping, const EnumStoreBase &enumStore); virtual ~MultiValueEnumAttributeSaver(); diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 5cea78df5b9..d36ccaec995 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -189,7 +189,7 @@ MultiValueNumericAttribute<B, M>::onInitSave() vespalib::GenerationHandler::Guard guard(this->getGenerationHandler(). takeGuard()); return std::make_unique<MultiValueNumericAttributeSaver<MultiValueType>> - (std::move(guard), this->createSaveTargetConfig(), this->_mvMapping); + (std::move(guard), this->createAttributeHeader(), this->_mvMapping); } template <typename B, typename M> diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp index 147eb18bdef..d2ebd95d9f6 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp @@ -44,9 +44,9 @@ public: template <typename MultiValueT> MultiValueNumericAttributeSaver<MultiValueT>:: MultiValueNumericAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, - const MultiValueMapping &mvMapping) - : Parent(std::move(guard), cfg, mvMapping), + const attribute::AttributeHeader &header, + const MultiValueMapping &mvMapping) + : Parent(std::move(guard), header, mvMapping), _mvMapping(mvMapping) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.h b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.h index 9b786ea5769..a2dd0225541 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.h @@ -26,7 +26,7 @@ class MultiValueNumericAttributeSaver : public MultiValueAttributeSaver public: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; MultiValueNumericAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const MultiValueMapping &mvMapping); virtual ~MultiValueNumericAttributeSaver(); diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp index a7d8045443c..0d5a7b465c4 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.cpp @@ -4,16 +4,15 @@ #include "multivalueattributesaver.h" using vespalib::GenerationHandler; -using search::IAttributeSaveTarget; namespace search { MultiValueAttributeSaver:: MultiValueAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const MvMappingBase &mvMapping) - : AttributeSaver(std::move(guard), cfg), - _frozenIndices(mvMapping.getRefCopy(cfg.getNumDocs())) + : AttributeSaver(std::move(guard), header), + _frozenIndices(mvMapping.getRefCopy(header.getNumDocs())) { } diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.h index e8b792cbb75..6bf755db143 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaver.h @@ -3,7 +3,6 @@ #pragma once #include "attributesaver.h" -#include "iattributesavetarget.h" #include "multi_value_mapping.h" namespace search { @@ -21,7 +20,7 @@ protected: public: MultiValueAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, const MvMappingBase &mvMapping); virtual ~MultiValueAttributeSaver(); diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h index fe10abfd435..8024d287a0d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h @@ -2,6 +2,8 @@ #pragma once +#include "iattributesavetarget.h" + namespace search { namespace multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp index 3d3bbdcae43..e7df13549ef 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp @@ -5,6 +5,8 @@ #include <vespa/document/fieldvalue/predicatefieldvalue.h> #include <vespa/document/predicate/predicate.h> #include <vespa/vespalib/data/slime/slime.h> +#include "iattributesavetarget.h" +#include "attribute_header.h" #include <vespa/log/log.h> LOG_SETUP(".predicate_attribute"); @@ -55,7 +57,7 @@ int64_t adjustUpperBound(int32_t arity, int64_t upper_bound) { } SimpleIndexConfig createSimpleIndexConfig(const search::attribute::Config &config) { - return SimpleIndexConfig(config.dense_posting_list_threshold(), config.getGrowStrategy()); + return SimpleIndexConfig(config.predicateParams().dense_posting_list_threshold(), config.getGrowStrategy()); } } // namespace @@ -66,9 +68,9 @@ PredicateAttribute::PredicateAttribute(const vespalib::string &base_file_name, _base_file_name(base_file_name), _limit_provider(*this), _index(new PredicateIndex(getGenerationHandler(), getGenerationHolder(), - _limit_provider, createSimpleIndexConfig(config), config.arity())), - _lower_bound(adjustLowerBound(config.arity(), config.lower_bound())), - _upper_bound(adjustUpperBound(config.arity(), config.upper_bound())), + _limit_provider, createSimpleIndexConfig(config), config.predicateParams().arity())), + _lower_bound(adjustLowerBound(config.predicateParams().arity(), config.predicateParams().lower_bound())), + _upper_bound(adjustUpperBound(config.predicateParams().arity(), config.predicateParams().upper_bound())), _min_feature(config.getGrowStrategy(), getGenerationHolder()), _interval_range_vector(config.getGrowStrategy(), getGenerationHolder()), _max_interval_range(1) @@ -183,8 +185,11 @@ bool PredicateAttribute::onLoad() buffer.moveFreeToData(size); const GenericHeader &header = loaded_buffer->getHeader(); - uint32_t version = static_cast<uint32_t>( - header.hasTag("version") ? header.getTag("version").asInteger() : 0); + auto attributeHeader = attribute::AttributeHeader::extractTags(header); + uint32_t version = attributeHeader.getVersion(); + + setCreateSerialNum(attributeHeader.getCreateSerialNum()); + LOG(info, "Loading predicate attribute version %d. getVersion() = %d", version, getVersion()); DocId highest_doc_id; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 629c7cf5b32..8909b854423 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -127,7 +127,7 @@ ReferenceAttribute::onInitSave() takeGuard()); return std::make_unique<ReferenceAttributeSaver> (std::move(guard), - createSaveTargetConfig(), + createAttributeHeader(), getIndicesCopy(getCommittedDocIdLimit()), _store); } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp index fcc1fc11f22..7af9f607451 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp @@ -3,6 +3,7 @@ #include "reference_attribute_saver.h" #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/array.hpp> +#include "iattributesavetarget.h" using vespalib::GenerationHandler; @@ -14,10 +15,10 @@ namespace attribute { ReferenceAttributeSaver:: ReferenceAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, IndicesCopyVector &&indices, const Store &store) - : AttributeSaver(std::move(guard), cfg), + : AttributeSaver(std::move(guard), header), _indices(std::move(indices)), _store(store), _saver(store.getSaver()) diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h index 012356f24f9..b1d4dcf3a79 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h @@ -7,7 +7,6 @@ #include <vespa/searchlib/datastore/unique_store.h> #include <vespa/searchlib/datastore/unique_store_saver.h> #include <vespa/searchlib/common/rcuvector.h> -#include "iattributesavetarget.h" #include "reference_attribute.h" namespace search { @@ -41,7 +40,7 @@ private: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; public: ReferenceAttributeSaver(vespalib::GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, IndicesCopyVector &&indices, const Store &store); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index 345c7227627..c798351211a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -310,7 +310,7 @@ SingleValueEnumAttribute<B>::onInitSave() takeGuard()); return std::make_unique<SingleValueEnumAttributeSaver> (std::move(guard), - this->createSaveTargetConfig(), + this->createAttributeHeader(), getIndicesCopy(this->getCommittedDocIdLimit()), this->_enumStore); } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp index 8cc351e0330..e11ee1dbb87 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp @@ -3,6 +3,7 @@ #include "singleenumattributesaver.h" #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/array.hpp> +#include "iattributesavetarget.h" using vespalib::GenerationHandler; @@ -11,10 +12,10 @@ namespace search { SingleValueEnumAttributeSaver:: SingleValueEnumAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, EnumIndexCopyVector &&indices, const EnumStoreBase &enumStore) - : AttributeSaver(std::move(guard), cfg), + : AttributeSaver(std::move(guard), header), _indices(std::move(indices)), _enumSaver(enumStore, false) { diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h index 7e7de3ef84e..b989194e1b0 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h @@ -3,7 +3,6 @@ #pragma once #include "attributesaver.h" -#include "iattributesavetarget.h" #include "singleenumattribute.h" #include "enumattributesaver.h" @@ -23,7 +22,7 @@ private: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; public: SingleValueEnumAttributeSaver(vespalib::GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, EnumIndexCopyVector &&indices, const EnumStoreBase &enumStore); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index cbd4bfe383f..4285c868f24 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -197,7 +197,7 @@ SingleValueNumericAttribute<B>::onInitSave() const uint32_t numDocs(this->getCommittedDocIdLimit()); assert(numDocs <= _data.size()); return std::make_unique<SingleValueNumericAttributeSaver> - (this->createSaveTargetConfig(), &_data[0], numDocs * sizeof(T)); + (this->createAttributeHeader(), &_data[0], numDocs * sizeof(T)); } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp index 3320dd977d2..fe8cdf1ca42 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.cpp @@ -2,9 +2,9 @@ #include <vespa/fastos/fastos.h> #include "singlenumericattributesaver.h" +#include "iattributesavetarget.h" using vespalib::GenerationHandler; -using search::IAttributeSaveTarget; namespace search { @@ -17,9 +17,9 @@ const uint32_t MIN_ALIGNMENT = 4096; SingleValueNumericAttributeSaver:: -SingleValueNumericAttributeSaver(const IAttributeSaveTarget::Config &cfg, +SingleValueNumericAttributeSaver(const attribute::AttributeHeader &header, const void *data, size_t size) - : AttributeSaver(vespalib::GenerationHandler::Guard(), cfg), + : AttributeSaver(vespalib::GenerationHandler::Guard(), header), _buf() { _buf = std::make_unique<BufferBuf>(size, MIN_ALIGNMENT); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.h b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.h index 585e5c49dab..8d8248ae74f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattributesaver.h @@ -22,7 +22,7 @@ private: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; public: - SingleValueNumericAttributeSaver(const IAttributeSaveTarget::Config &cfg, + SingleValueNumericAttributeSaver(const attribute::AttributeHeader &header, const void *data, size_t size); virtual ~SingleValueNumericAttributeSaver(); diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index 8d75f00c901..83e9d6fdc92 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -5,6 +5,7 @@ #include "primitivereader.h" #include "attributeiterators.hpp" #include <vespa/searchlib/queryeval/emptysearch.h> +#include "iattributesavetarget.h" namespace search { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 7a395e04c09..731e51abbca 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -169,7 +169,7 @@ DenseTensorAttribute::onInitSave() takeGuard()); return std::make_unique<DenseTensorAttributeSaver> (std::move(guard), - this->createSaveTargetConfig(), + this->createAttributeHeader(), getRefCopy(), _denseTensorStore); } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp index d57d614c82c..7daec1159ca 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -4,9 +4,9 @@ #include "dense_tensor_attribute_saver.h" #include <vespa/searchlib/util/bufferwriter.h> #include "dense_tensor_store.h" +#include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; -using search::IAttributeSaveTarget; namespace search { @@ -21,10 +21,10 @@ static const uint8_t tensorIsPresent = 1; DenseTensorAttributeSaver:: DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, - RefCopyVector &&refs, - const DenseTensorStore &tensorStore) - : AttributeSaver(std::move(guard), cfg), + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const DenseTensorStore &tensorStore) + : AttributeSaver(std::move(guard), header), _refs(std::move(refs)), _tensorStore(tensorStore) { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h index 0b76cc9cfad..e297ef8edc6 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchlib/attribute/attributesaver.h> -#include <vespa/searchlib/attribute/iattributesavetarget.h> #include "tensor_attribute.h" namespace search { @@ -27,7 +26,7 @@ private: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; public: DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, RefCopyVector &&refs, const DenseTensorStore &tensorStore); diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp index 4fe24608b1e..627f2428e5f 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute.cpp @@ -103,7 +103,7 @@ GenericTensorAttribute::onInitSave() takeGuard()); return std::make_unique<GenericTensorAttributeSaver> (std::move(guard), - this->createSaveTargetConfig(), + this->createAttributeHeader(), getRefCopy(), _genericTensorStore); } diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp index ebcec991c73..6c22434afe9 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp @@ -4,9 +4,9 @@ #include "generic_tensor_attribute_saver.h" #include <vespa/searchlib/util/bufferwriter.h> #include "generic_tensor_store.h" +#include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; -using search::IAttributeSaveTarget; namespace search { @@ -14,10 +14,10 @@ namespace tensor { GenericTensorAttributeSaver:: GenericTensorAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, + const attribute::AttributeHeader &header, RefCopyVector &&refs, const GenericTensorStore &tensorStore) - : AttributeSaver(std::move(guard), cfg), + : AttributeSaver(std::move(guard), header), _refs(std::move(refs)), _tensorStore(tensorStore) { diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h index ad9d5550d3a..c351ae573b1 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.h @@ -3,7 +3,6 @@ #pragma once #include <vespa/searchlib/attribute/attributesaver.h> -#include <vespa/searchlib/attribute/iattributesavetarget.h> #include "tensor_attribute.h" namespace search { @@ -27,9 +26,9 @@ private: virtual bool onSave(IAttributeSaveTarget &saveTarget) override; public: GenericTensorAttributeSaver(GenerationHandler::Guard &&guard, - const IAttributeSaveTarget::Config &cfg, - RefCopyVector &&refs, - const GenericTensorStore &tensorStore); + const attribute::AttributeHeader &header, + RefCopyVector &&refs, + const GenericTensorStore &tensorStore); virtual ~GenericTensorAttributeSaver(); }; |