diff options
9 files changed, 189 insertions, 8 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 7c96a567f55..d7edca0ac35 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -62,6 +62,7 @@ vespa_define_module( src/tests/grouping src/tests/proton/attribute src/tests/proton/attribute/attribute_directory + src/tests/proton/attribute/attribute_initializer src/tests/proton/attribute/attribute_manager src/tests/proton/attribute/attribute_populator src/tests/proton/attribute/attribute_usage_filter diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt new file mode 100644 index 00000000000..ad1383f6247 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_initializer_test_app TEST + SOURCES + attribute_initializer_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attribute_initializer_test_app COMMAND searchcore_attribute_initializer_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/DESC b/searchcore/src/tests/proton/attribute/attribute_initializer/DESC new file mode 100644 index 00000000000..d7e2c61d85d --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/DESC @@ -0,0 +1 @@ +attribute initializer test. Take a look at attribute_initializer_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/FILES b/searchcore/src/tests/proton/attribute/attribute_initializer/FILES new file mode 100644 index 00000000000..cedbcb0155b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/FILES @@ -0,0 +1 @@ +attribute_initializer_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp new file mode 100644 index 00000000000..cb9579d15b6 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -0,0 +1,153 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_initializer_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/attribute/attributedisklayout.h> +#include <vespa/searchcore/proton/attribute/attribute_directory.h> +#include <vespa/searchcore/proton/attribute/attribute_factory.h> +#include <vespa/searchcore/proton/attribute/attribute_initializer.h> + +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::SerialNum; + +const vespalib::string test_dir = "test_output"; + +namespace proton +{ + +namespace { + +const Config predicate(BasicType::Type::PREDICATE); + +Config getPredicate2(uint32_t arity) +{ + Config ret(predicate); + search::attribute::PredicateParams predicateParams; + predicateParams.setArity(arity); + ret.setPredicateParams(predicateParams); + return ret; +} + +Config getTensor(const vespalib::string &spec) +{ + Config ret(BasicType::Type::TENSOR); + ret.setTensorType(vespalib::eval::ValueType::from_spec(spec)); + return ret; +} + +void +saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, SerialNum createSerialNum) +{ + auto diskLayout = AttributeDiskLayout::create(test_dir); + auto dir = diskLayout->createAttributeDir(name); + auto writer = dir->getWriter(); + writer->createInvalidSnapshot(serialNum); + auto snapshotdir = writer->getSnapshotDir(serialNum); + vespalib::mkdir(snapshotdir); + auto av = search::AttributeFactory::createAttribute(snapshotdir + "/" + name, cfg); + av->setCreateSerialNum(createSerialNum); + av->addReservedDoc(); + uint32_t docId; + av->addDoc(docId); + assert(docId == 1u); + av->clearDoc(docId); + av->save(); + writer->markValidSnapshot(serialNum); +} + +} + +struct Fixture +{ + test::DirectoryHandler _dirHandler; + std::shared_ptr<AttributeDiskLayout> _diskLayout; + AttributeFactory _factory; + Fixture(); + ~Fixture(); + std::unique_ptr<AttributeInitializer> createInitializer(const vespalib::string &name, const Config &cfg, SerialNum serialNum); +}; + +Fixture::Fixture() + : _dirHandler(test_dir), + _diskLayout(AttributeDiskLayout::create(test_dir)), + _factory() +{ +} + +Fixture::~Fixture() {} + +std::unique_ptr<AttributeInitializer> +Fixture::createInitializer(const vespalib::string &name, const Config &cfg, SerialNum serialNum) +{ + return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(name), "test.subdb", cfg, serialNum, _factory); +} + + +TEST("require that predicate attributes can be initialized") +{ + saveAttr("a", predicate, 10, 2); + Fixture f; + auto av = f.createInitializer("a", predicate, 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that predicate attributes will not be initialized with future-created attribute") +{ + saveAttr("a", predicate, 10, 8); + Fixture f; + auto av = f.createInitializer("a", predicate, 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that predicate attributes will not be initialized with mismatching type") +{ + saveAttr("a", predicate, 10, 2); + Fixture f; + auto av = f.createInitializer("a", getPredicate2(4), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that tensor attribute can be initialized") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 2); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[10])"), 5)->init(); + EXPECT_EQUAL(2, av->getCreateSerialNum()); + EXPECT_EQUAL(2, av->getNumDocs()); +} + +TEST("require that tensor attributes will not be initialized with future-created attribute") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 8); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[10])"), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +TEST("require that tensor attributes will not be initialized with mismatching type") +{ + saveAttr("a", getTensor("tensor(x[10])"), 10, 2); + Fixture f; + auto av = f.createInitializer("a", getTensor("tensor(x[11])"), 5)->init(); + EXPECT_EQUAL(5, av->getCreateSerialNum()); + EXPECT_EQUAL(1, av->getNumDocs()); +} + +} + +TEST_MAIN() +{ + vespalib::rmdir(test_dir, true); + TEST_RUN_ALL(); +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 725a671e7ab..7a23db1a071 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -86,9 +86,7 @@ extractHeader(const vespalib::string &attrFileName) auto df = search::FileUtil::openFile(attrFileName + ".dat"); vespalib::FileHeader datHeader; datHeader.readFile(*df); - AttributeHeader retval; - retval.extractTags(datHeader); - return retval; + return AttributeHeader::extractTags(datHeader); } void diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 933b62abf68..84f0720b140 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -8,6 +8,7 @@ namespace attribute { namespace { +const vespalib::string versionTag = "version"; const vespalib::string dataTypeTag = "datatype"; const vespalib::string collectionTypeTag = "collectiontype"; const vespalib::string createSerialNumTag = "createSerialNum"; @@ -68,7 +69,7 @@ AttributeHeader::~AttributeHeader() } void -AttributeHeader::extractTags(const vespalib::GenericHeader &header) +AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header) { if (header.hasTag(createSerialNumTag)) { _createSerialNum = header.getTag(createSerialNumTag).asInteger(); @@ -95,6 +96,17 @@ AttributeHeader::extractTags(const vespalib::GenericHeader &header) assert(!header.hasTag(predicateUpperBoundTag)); } } + if (header.hasTag(versionTag)) { + _version = header.getTag(versionTag).asInteger(); + } +} + +AttributeHeader +AttributeHeader::extractTags(const vespalib::GenericHeader &header) +{ + AttributeHeader result; + result.internalExtractTags(header); + return result; } void @@ -108,7 +120,7 @@ AttributeHeader::addTags(vespalib::GenericHeader &header) const header.putTag(Tag("docIdLimit", _numDocs)); header.putTag(Tag("frozen", 0)); header.putTag(Tag("fileBitSize", 0)); - header.putTag(Tag("version", _version)); + header.putTag(Tag(versionTag, _version)); if (_enumerated) { header.putTag(Tag("enumerated", 1)); } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.h b/searchlib/src/vespa/searchlib/attribute/attribute_header.h index 32f6123036e..7e9ef78dafb 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.h @@ -32,6 +32,8 @@ private: uint64_t _totalValueCount; uint64_t _createSerialNum; uint32_t _version; + + void internalExtractTags(const vespalib::GenericHeader &header); public: AttributeHeader(); AttributeHeader(const vespalib::string &fileName, @@ -63,7 +65,7 @@ public: uint32_t getVersion() const { return _version; } const PersistentPredicateParams &getPredicateParams() const { return _predicateParams; } bool getPredicateParamsSet() const { return _predicateParamsSet; } - void extractTags(const vespalib::GenericHeader &header); + static AttributeHeader extractTags(const vespalib::GenericHeader &header); void addTags(vespalib::GenericHeader &header) const; }; diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp index 4ce7015fdba..e7df13549ef 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp @@ -6,6 +6,7 @@ #include <vespa/document/predicate/predicate.h> #include <vespa/vespalib/data/slime/slime.h> #include "iattributesavetarget.h" +#include "attribute_header.h" #include <vespa/log/log.h> LOG_SETUP(".predicate_attribute"); @@ -184,8 +185,11 @@ bool PredicateAttribute::onLoad() buffer.moveFreeToData(size); const GenericHeader &header = loaded_buffer->getHeader(); - uint32_t version = static_cast<uint32_t>( - header.hasTag("version") ? header.getTag("version").asInteger() : 0); + auto attributeHeader = attribute::AttributeHeader::extractTags(header); + uint32_t version = attributeHeader.getVersion(); + + setCreateSerialNum(attributeHeader.getCreateSerialNum()); + LOG(info, "Loading predicate attribute version %d. getVersion() = %d", version, getVersion()); DocId highest_doc_id; |