diff options
Diffstat (limited to 'searchcore/src/tests/proton')
8 files changed, 284 insertions, 2 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt new file mode 100644 index 00000000000..ad1383f6247 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_initializer_test_app TEST + SOURCES + attribute_initializer_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attribute_initializer_test_app COMMAND searchcore_attribute_initializer_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/DESC b/searchcore/src/tests/proton/attribute/attribute_initializer/DESC new file mode 100644 index 00000000000..d7e2c61d85d --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/DESC @@ -0,0 +1 @@ +attribute initializer test. Take a look at attribute_initializer_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/FILES b/searchcore/src/tests/proton/attribute/attribute_initializer/FILES new file mode 100644 index 00000000000..cedbcb0155b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/FILES @@ -0,0 +1 @@ +attribute_initializer_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp new file mode 100644 index 00000000000..37ef21fb700 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -0,0 +1,182 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_initializer_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/attribute/attribute_directory.h> +#include <vespa/searchcore/proton/attribute/attribute_factory.h> +#include <vespa/searchcore/proton/attribute/attribute_initializer.h> + +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::SerialNum; + +const vespalib::string test_dir = "test_output"; + +namespace proton +{ + +namespace { + +const Config int32_sv(BasicType::Type::INT32); +const Config int16_sv(BasicType::Type::INT16); +const Config int32_array(BasicType::Type::INT32, CollectionType::Type::ARRAY); +const Config predicate(BasicType::Type::PREDICATE); + +Config getPredicateWithArity(uint32_t arity) +{ + Config ret(predicate); + search::attribute::PredicateParams predicateParams; + predicateParams.setArity(arity); + ret.setPredicateParams(predicateParams); + return ret; +} + +Config getTensor(const vespalib::string &spec) +{ + Config ret(BasicType::Type::TENSOR); + ret.setTensorType(vespalib::eval::ValueType::from_spec(spec)); + return ret; +} + +void +saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, SerialNum createSerialNum) +{ + auto diskLayout = AttributeDiskLayout::create(test_dir); + auto dir = diskLayout->createAttributeDir(name); + auto writer = dir->getWriter(); + writer->createInvalidSnapshot(serialNum); + auto snapshotdir = writer->getSnapshotDir(serialNum); + vespalib::mkdir(snapshotdir); + auto av = search::AttributeFactory::createAttribute(snapshotdir + "/" + name, cfg); + av->setCreateSerialNum(createSerialNum); + av->addReservedDoc(); + uint32_t docId; + av->addDoc(docId); + assert(docId == 1u); + av->clearDoc(docId); + av->save(); + writer->markValidSnapshot(serialNum); +} + +} + +struct Fixture +{ + test::DirectoryHandler _dirHandler; + std::shared_ptr<AttributeDiskLayout> _diskLayout; + AttributeFactory _factory; + Fixture(); + ~Fixture(); + std::unique_ptr<AttributeInitializer> createInitializer(const vespalib::string &name, const Config &cfg, SerialNum serialNum); +}; + +Fixture::Fixture() + : _dirHandler(test_dir), + _diskLayout(AttributeDiskLayout::create(test_dir)), + _factory() +{ +} + +Fixture::~Fixture() {} + +std::unique_ptr<AttributeInitializer> +Fixture::createInitializer(const vespalib::string &name, const Config &cfg, SerialNum serialNum) +{ + return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(name), "test.subdb", cfg, serialNum, _factory); +} + +TEST("require that integer attribute can be initialized") +{ + saveAttr("a", int32_sv, 10, 2); + Fixture f; + auto av = f.createInitializer("a", int32_sv, 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that mismatching base type is not loaded") +{ + saveAttr("a", int32_sv, 10, 2); + Fixture f; + auto av = f.createInitializer("a", int16_sv, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that mismatching collection type is not loaded") +{ + saveAttr("a", int32_sv, 10, 2); + Fixture f; + auto av = f.createInitializer("a", int32_array, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that predicate attributes can be initialized") +{ + saveAttr("a", predicate, 10, 2); + Fixture f; + auto av = f.createInitializer("a", predicate, 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that predicate attributes will not be initialized with future-created attribute") +{ + saveAttr("a", predicate, 10, 8); + Fixture f; + auto av = f.createInitializer("a", predicate, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that predicate attributes will not be initialized with mismatching type") +{ + saveAttr("a", predicate, 10, 2); + Fixture f; + auto av = f.createInitializer("a", getPredicateWithArity(4), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that tensor attribute can be initialized") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 2); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[10])"), 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that tensor attributes will not be initialized with future-created attribute") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 8); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[10])"), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that tensor attributes will not be initialized with mismatching type") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 2); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[11])"), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +} + +TEST_MAIN() +{ + vespalib::rmdir(test_dir, true); + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index a030c65954d..7232f997b0e 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -61,6 +61,7 @@ using search::predicate::PredicateIndex; using search::predicate::PredicateTreeAnnotations; using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; +using vespalib::eval::ValueType; typedef search::attribute::Config AVConfig; typedef proton::AttributeCollectionSpec::Attribute AttrSpec; @@ -111,6 +112,19 @@ fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64 test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); } +search::SerialNum getCreateSerialNum(const AttributeGuard::UP &guard) +{ + if (!guard || !guard->valid()) { + return 0; + } else { + return (*guard)->getCreateSerialNum(); + } +} + +void assertCreateSerialNum(const AttributeManager &am, const vespalib::string &name, search::SerialNum expCreateSerialNum) { + EXPECT_EQUAL(expCreateSerialNum, getCreateSerialNum(am.getAttribute(name))); +} + struct ImportedAttributesRepoBuilder { ImportedAttributesRepo::UP _repo; ImportedAttributesRepoBuilder() : _repo(std::make_unique<ImportedAttributesRepo>()) {} @@ -740,6 +754,53 @@ TEST_F("require that imported attributes are exposed via attribute context toget EXPECT_EQUAL("imported", all[1]->getName()); } +TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) +{ + AVConfig generic_tensor(BasicType::TENSOR); + generic_tensor.setTensorType(ValueType::tensor_type({})); + AVConfig dense_tensor(BasicType::TENSOR); + dense_tensor.setTensorType(ValueType::from_spec("tensor(x[10])")); + AVConfig predicate(BasicType::PREDICATE); + using PredicateParams = search::attribute::PredicateParams; + PredicateParams predicateParams; + predicateParams.setArity(2); + predicate.setPredicateParams(predicateParams); + AVConfig predicate2(BasicType::PREDICATE); + PredicateParams predicateParams2; + predicateParams2.setArity(4); + predicate2.setPredicateParams(predicateParams2); + + auto am1(std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), + f._fileHeaderContext, f._attributeFieldWriter, f._hwInfo)); + am1->addAttribute("a1", INT32_SINGLE, 1); + am1->addAttribute("a2", INT32_SINGLE, 2); + am1->addAttribute("a3", generic_tensor, 3); + am1->addAttribute("a4", generic_tensor, 4); + am1->addAttribute("a5", predicate, 5); + am1->addAttribute("a6", predicate, 6); + AttrSpecList newSpec; + newSpec.push_back(AttrSpec("a1", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a2", INT32_ARRAY)); + newSpec.push_back(AttrSpec("a3", generic_tensor)); + newSpec.push_back(AttrSpec("a4", dense_tensor)); + newSpec.push_back(AttrSpec("a5", predicate)); + newSpec.push_back(AttrSpec("a6", predicate2)); + SequentialAttributeManager am2(*am1, AttrMgrSpec(newSpec, 5, 20)); + TEST_DO(assertCreateSerialNum(*am1, "a1", 1)); + TEST_DO(assertCreateSerialNum(*am1, "a2", 2)); + TEST_DO(assertCreateSerialNum(*am1, "a3", 3)); + TEST_DO(assertCreateSerialNum(*am1, "a4", 4)); + TEST_DO(assertCreateSerialNum(*am1, "a5", 5)); + TEST_DO(assertCreateSerialNum(*am1, "a6", 6)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a1", 1)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a2", 20)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a3", 3)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a4", 20)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a5", 5)); + TEST_DO(assertCreateSerialNum(am2.mgr, "a6", 20)); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index e28dd24a454..19bfedfcd31 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -68,7 +68,7 @@ struct Fixture _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), _fileHeader, _attributeFieldWriter, _hwInfo)), - _pop(_mgr, 1, "test"), + _pop(_mgr, 1, "test", CREATE_SERIAL_NUM), _ctx() { _mgr->addAttribute("a1", AVConfig(AVBasicType::INT32), @@ -93,6 +93,8 @@ TEST_F("require that reprocess with document populates attribute", Fixture) EXPECT_EQUAL(7u, attr->get()->getNumDocs()); EXPECT_EQUAL(44, attr->get()->getInt(6)); EXPECT_EQUAL(2u, attr->get()->getStatus().getLastSyncToken()); + f._pop.done(); + EXPECT_EQUAL(CREATE_SERIAL_NUM, attr->get()->getStatus().getLastSyncToken()); } TEST_MAIN() diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 36319ae77d5..f560030db97 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -5,12 +5,15 @@ LOG_SETUP("attribute_reprocessing_initializer_test"); #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchcore/proton/test/directory_handler.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -19,6 +22,8 @@ LOG_SETUP("attribute_reprocessing_initializer_test"); using namespace proton; using namespace search; using namespace search::index; +using search::attribute::Config; +using search::attribute::BasicType; const vespalib::string TEST_DIR = "test_output"; const SerialNum INIT_SERIAL_NUM = 10; @@ -118,10 +123,11 @@ struct Fixture _handler() { } + ~Fixture() { } void init() { _initializer.reset(new AttributeReprocessingInitializer (ARIConfig(_newCfg._mgr, _newCfg._schema, _newCfg._inspector), - ARIConfig(_oldCfg._mgr, _oldCfg._schema, _oldCfg._inspector), "test")); + ARIConfig(_oldCfg._mgr, _oldCfg._schema, _oldCfg._inspector), "test", INIT_SERIAL_NUM)); _initializer->initialize(_handler); } Fixture &addOldConfig(const StringVector &fields, @@ -249,6 +255,25 @@ TEST_F("require that predicate fields are not populated from attribute", Fixture EXPECT_TRUE(f.assertFields({"c", "d"})); } +TEST("require that added attribute aspect with flushed attribute after interruptted reprocessing does not require attribute populate") +{ + { + auto diskLayout = AttributeDiskLayout::create(TEST_DIR); + auto dir = diskLayout->createAttributeDir("a"); + auto writer = dir->getWriter(); + writer->createInvalidSnapshot(INIT_SERIAL_NUM); + writer->markValidSnapshot(INIT_SERIAL_NUM); + auto snapshotdir = writer->getSnapshotDir(INIT_SERIAL_NUM); + vespalib::mkdir(snapshotdir); + auto av = AttributeFactory::createAttribute(snapshotdir + "/a", + Config(BasicType::STRING)); + av->save(); + } + Fixture f; + f.addOldConfig({"a"}, {}).addNewConfig({"a"}, {"a"}).init(); + EXPECT_TRUE(f.assertAttributes({})); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp index 483c1c3b977..1848ad8fc74 100644 --- a/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/document_reprocessing_handler/document_reprocessing_handler_test.cpp @@ -23,6 +23,7 @@ struct MyProcessor : public ReprocessingType _lid = lid; _docId = doc.getId(); } + virtual void done() { } }; typedef MyProcessor<IReprocessingReader, const Document &> MyReader; |