diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /searchcore/src/tests/proton/attribute |
Publish
Diffstat (limited to 'searchcore/src/tests/proton/attribute')
39 files changed, 2420 insertions, 0 deletions
diff --git a/searchcore/src/tests/proton/attribute/.gitignore b/searchcore/src/tests/proton/attribute/.gitignore new file mode 100644 index 00000000000..794f5f454f8 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/.gitignore @@ -0,0 +1,9 @@ +.depend +Makefile +*_test +test +test_output +flush + +searchcore_attribute_test_app +searchcore_attributeflush_test_app diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt new file mode 100644 index 00000000000..1439c2b2646 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_test_app + SOURCES + attribute_test.cpp + DEPENDS + searchcore_server + searchcore_attribute + searchcore_flushengine + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attribute_test_app COMMAND sh attribute_test.sh) +vespa_add_executable(searchcore_attributeflush_test_app + SOURCES + attributeflush_test.cpp + DEPENDS + searchcore_server + searchcore_attribute + searchcore_flushengine + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attributeflush_test_app COMMAND sh attributeflush_test.sh) diff --git a/searchcore/src/tests/proton/attribute/DESC b/searchcore/src/tests/proton/attribute/DESC new file mode 100644 index 00000000000..bd71a808c51 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/DESC @@ -0,0 +1 @@ +attribute test. Take a look at attribute.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/FILES b/searchcore/src/tests/proton/attribute/FILES new file mode 100644 index 00000000000..84bc710d58b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/FILES @@ -0,0 +1 @@ +attribute.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore b/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore new file mode 100644 index 00000000000..3e77da66466 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_manager/.gitignore @@ -0,0 +1 @@ +searchcore_attribute_manager_test_app diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt new file mode 100644 index 00000000000..7e8ab14a13b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_manager/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_manager_test_app + SOURCES + attribute_manager_test.cpp + DEPENDS + searchcore_server + searchcore_attribute + searchcore_documentmetastore + searchcore_bucketdb + searchcore_initializer + searchcore_flushengine + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attribute_manager_test_app COMMAND searchcore_attribute_manager_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/DESC b/searchcore/src/tests/proton/attribute/attribute_manager/DESC new file mode 100644 index 00000000000..f1cdc01fd47 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_manager/DESC @@ -0,0 +1 @@ +attribute manager test. Take a look at attribute_manager_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/FILES b/searchcore/src/tests/proton/attribute/attribute_manager/FILES new file mode 100644 index 00000000000..8e4fbdcb888 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_manager/FILES @@ -0,0 +1 @@ +attribute_manager_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp new file mode 100644 index 00000000000..34c67da4ac8 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -0,0 +1,686 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_manager_test"); + +#include <vespa/fastos/file.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h> +#include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h> +#include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> +#include <vespa/searchcore/proton/attribute/i_attribute_functor.h> +#include <vespa/searchcore/proton/initializer/initializer_task.h> +#include <vespa/searchcore/proton/initializer/task_runner.h> +#include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/test/attribute_vectors.h> +#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/util/filekit.h> + +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/predicate_attribute.h> +#include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/predicate/predicate_tree_annotator.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/config-attributes.h> + +namespace vespa { namespace config { namespace search {}}} + +using std::string; +using namespace vespa::config::search; +using namespace config; +using namespace document; +using namespace proton; +using namespace search; +using namespace search::index; +using proton::initializer::InitializerTask; +using proton::test::AttributeUtils; +using proton::test::Int32Attribute; +using search::TuneFileAttributes; +using search::index::DummyFileHeaderContext; +using search::ForegroundTaskExecutor; +using search::predicate::PredicateIndex; +using search::predicate::PredicateTreeAnnotations; +using vespa::config::search::AttributesConfig; +using vespa::config::search::AttributesConfigBuilder; + +typedef search::attribute::Config AVConfig; +typedef proton::AttributeCollectionSpec::Attribute AttrSpec; +typedef proton::AttributeCollectionSpec::AttributeList AttrSpecList; +typedef proton::AttributeCollectionSpec AttrMgrSpec; + +namespace { + +const uint64_t createSerialNum = 42u; + +class MyAttributeFunctor : public proton::IAttributeFunctor +{ + std::vector<vespalib::string> _names; + +public: + virtual void + operator()(const search::AttributeVector &attributeVector) override { + _names.push_back(attributeVector.getName()); + } + + std::string getSortedNames() { + std::ostringstream os; + std::sort(_names.begin(), _names.end()); + for (const vespalib::string &name : _names) { + if (!os.str().empty()) + os << ","; + os << name; + } + return os.str(); + } +}; + +} + +const string test_dir = "test_output"; +const AVConfig INT32_SINGLE = AttributeUtils::getInt32Config(); +const AVConfig INT32_ARRAY = AttributeUtils::getInt32ArrayConfig(); + +void +fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) +{ + test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); +} + +void +fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) +{ + test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); +} + +struct BaseFixture +{ + test::DirectoryHandler _dirHandler; + DummyFileHeaderContext _fileHeaderContext; + ForegroundTaskExecutor _attributeFieldWriter; + BaseFixture() + : _dirHandler(test_dir), + _fileHeaderContext(), + _attributeFieldWriter() + { + } +}; + + +struct AttributeManagerFixture +{ + proton::AttributeManager::SP _msp; + proton::AttributeManager &_m; + AttributeWriter _aw; + AttributeManagerFixture(BaseFixture &bf) + : _msp(std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext, + bf._attributeFieldWriter)), + _m(*_msp), + _aw(_msp) + { + } + AttributeVector::SP addAttribute(const vespalib::string &name) { + return _m.addAttribute(name, INT32_SINGLE, createSerialNum); + } +}; + +struct Fixture : public BaseFixture, public AttributeManagerFixture +{ + Fixture() + : BaseFixture(), + AttributeManagerFixture(*static_cast<BaseFixture *>(this)) + { + } +}; + +struct SequentialAttributeManager +{ + SequentialAttributesInitializer initializer; + proton::AttributeManager mgr; + SequentialAttributeManager(const AttributeManager &currMgr, + const AttrMgrSpec &newSpec) + : initializer(newSpec.getDocIdLimit()), + mgr(currMgr, newSpec, initializer) + { + mgr.addInitializedAttributes(initializer.getInitializedAttributes()); + } +}; + +struct DummyInitializerTask : public InitializerTask +{ + virtual void run() override {} +}; + +struct ParallelAttributeManager +{ + InitializerTask::SP documentMetaStoreInitTask; + BucketDBOwner::SP bucketDbOwner; + DocumentMetaStore::SP documentMetaStore; + search::GrowStrategy attributeGrow; + size_t attributeGrowNumDocs; + bool fastAccessAttributesOnly; + std::shared_ptr<AttributeManager::SP> mgr; + AttributeManagerInitializer::SP initializer; + + ParallelAttributeManager(search::SerialNum configSerialNum, + AttributeManager::SP baseAttrMgr, + const AttributesConfig &attrCfg, + uint32_t docIdLimit) + : documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()), + bucketDbOwner(std::make_shared<BucketDBOwner>()), + documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)), + attributeGrow(), + attributeGrowNumDocs(1), + fastAccessAttributesOnly(false), + mgr(std::make_shared<AttributeManager::SP>()), + initializer(std::make_shared<AttributeManagerInitializer> + (configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg, + attributeGrow, attributeGrowNumDocs, fastAccessAttributesOnly, mgr)) + { + documentMetaStore->setCommittedDocIdLimit(docIdLimit); + vespalib::ThreadStackExecutor executor(3, 128 * 1024); + initializer::TaskRunner taskRunner(executor); + taskRunner.runTask(initializer); + } +}; + + +TEST_F("require that attributes are added", Fixture) +{ + EXPECT_TRUE(f.addAttribute("a1").get() != NULL); + EXPECT_TRUE(f.addAttribute("a2").get() != NULL); + EXPECT_EQUAL("a1", (*f._m.getAttribute("a1"))->getName()); + EXPECT_EQUAL("a1", (*f._m.getAttributeStableEnum("a1"))->getName()); + EXPECT_EQUAL("a2", (*f._m.getAttribute("a2"))->getName()); + EXPECT_EQUAL("a2", (*f._m.getAttributeStableEnum("a2"))->getName()); + EXPECT_TRUE(!f._m.getAttribute("not")->valid()); +} + +TEST_F("require that predicate attributes are added", Fixture) +{ + EXPECT_TRUE(f._m.addAttribute("p1", AttributeUtils::getPredicateConfig(), + createSerialNum).get() != NULL); + EXPECT_EQUAL("p1", (*f._m.getAttribute("p1"))->getName()); + EXPECT_EQUAL("p1", (*f._m.getAttributeStableEnum("p1"))->getName()); +} + +TEST_F("require that attributes are flushed and loaded", BaseFixture) +{ + IndexMetaInfo ia1(test_dir + "/a1"); + IndexMetaInfo ia2(test_dir + "/a2"); + IndexMetaInfo ia3(test_dir + "/a3"); + { + AttributeManagerFixture amf(f); + proton::AttributeManager &am = amf._m; + AttributeVector::SP a1 = amf.addAttribute("a1"); + EXPECT_EQUAL(1u, a1->getNumDocs()); // Resized to size of attributemanager + fillAttribute(a1, 1, 3, 2, 10); + EXPECT_EQUAL(3u, a1->getNumDocs()); // Resized to size of attributemanager + AttributeVector::SP a2 = amf.addAttribute("a2"); + EXPECT_EQUAL(1u, a2->getNumDocs()); // Not resized to size of attributemanager + fillAttribute(a2, 1, 5, 4, 10); + EXPECT_EQUAL(5u, a2->getNumDocs()); // Increased + EXPECT_TRUE(ia1.load()); + EXPECT_TRUE(!ia1.getBestSnapshot().valid); + EXPECT_TRUE(ia2.load()); + EXPECT_TRUE(!ia2.getBestSnapshot().valid); + EXPECT_TRUE(!ia3.load()); + am.flushAll(0); + EXPECT_TRUE(ia1.load()); + EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + EXPECT_TRUE(ia2.load()); + EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken); + } + { + AttributeManagerFixture amf(f); + proton::AttributeManager &am = amf._m; + AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded + + EXPECT_EQUAL(3u, a1->getNumDocs()); + fillAttribute(a1, 1, 2, 20); + EXPECT_EQUAL(4u, a1->getNumDocs()); + AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded + EXPECT_EQUAL(5u, a2->getNumDocs()); + EXPECT_EQUAL(4u, a1->getNumDocs()); + amf._aw.onReplayDone(5u); + EXPECT_EQUAL(5u, a2->getNumDocs()); + EXPECT_EQUAL(5u, a1->getNumDocs()); + fillAttribute(a2, 1, 4, 20); + EXPECT_EQUAL(6u, a2->getNumDocs()); + AttributeVector::SP a3 = amf.addAttribute("a3"); // not-loaded + EXPECT_EQUAL(1u, a3->getNumDocs()); + amf._aw.onReplayDone(6); + EXPECT_EQUAL(6u, a3->getNumDocs()); + fillAttribute(a3, 1, 7, 6, 20); + EXPECT_EQUAL(7u, a3->getNumDocs()); + EXPECT_TRUE(ia1.load()); + EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + EXPECT_TRUE(ia2.load()); + EXPECT_EQUAL(10u, ia2.getBestSnapshot().syncToken); + EXPECT_TRUE(ia3.load()); + EXPECT_TRUE(!ia3.getBestSnapshot().valid); + am.flushAll(0); + EXPECT_TRUE(ia1.load()); + EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken); + EXPECT_TRUE(ia2.load()); + EXPECT_EQUAL(20u, ia2.getBestSnapshot().syncToken); + EXPECT_TRUE(ia3.load()); + EXPECT_EQUAL(20u, ia3.getBestSnapshot().syncToken); + } + { + AttributeManagerFixture amf(f); + AttributeVector::SP a1 = amf.addAttribute("a1"); // loaded + EXPECT_EQUAL(6u, a1->getNumDocs()); + AttributeVector::SP a2 = amf.addAttribute("a2"); // loaded + EXPECT_EQUAL(6u, a1->getNumDocs()); + EXPECT_EQUAL(6u, a2->getNumDocs()); + AttributeVector::SP a3 = amf.addAttribute("a3"); // loaded + EXPECT_EQUAL(6u, a1->getNumDocs()); + EXPECT_EQUAL(6u, a2->getNumDocs()); + EXPECT_EQUAL(7u, a3->getNumDocs()); + amf._aw.onReplayDone(7); + EXPECT_EQUAL(7u, a1->getNumDocs()); + EXPECT_EQUAL(7u, a2->getNumDocs()); + EXPECT_EQUAL(7u, a3->getNumDocs()); + } +} + +TEST_F("require that predicate attributes are flushed and loaded", BaseFixture) +{ + IndexMetaInfo ia1(test_dir + "/a1"); + { + AttributeManagerFixture amf(f); + proton::AttributeManager &am = amf._m; + AttributeVector::SP a1 = + am.addAttribute("a1", + AttributeUtils::getPredicateConfig(), + createSerialNum); + EXPECT_EQUAL(1u, a1->getNumDocs()); + + PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1); + PredicateIndex &index = pa.getIndex(); + uint32_t doc_id; + a1->addDoc(doc_id); + index.indexEmptyDocument(doc_id); + pa.commit(10, 10); + + EXPECT_EQUAL(2u, a1->getNumDocs()); + + EXPECT_TRUE(ia1.load()); + EXPECT_TRUE(!ia1.getBestSnapshot().valid); + am.flushAll(0); + EXPECT_TRUE(ia1.load()); + EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + } + { + AttributeManagerFixture amf(f); + proton::AttributeManager &am = amf._m; + AttributeVector::SP a1 = + am.addAttribute("a1", AttributeUtils::getPredicateConfig(), + createSerialNum); // loaded + EXPECT_EQUAL(2u, a1->getNumDocs()); + + PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1); + PredicateIndex &index = pa.getIndex(); + uint32_t doc_id; + a1->addDoc(doc_id); + PredicateTreeAnnotations annotations(3); + annotations.interval_map[123] = {{ 0x0001ffff }}; + index.indexDocument(1, annotations); + pa.commit(20, 20); + + EXPECT_EQUAL(3u, a1->getNumDocs()); + EXPECT_TRUE(ia1.load()); + EXPECT_EQUAL(10u, ia1.getBestSnapshot().syncToken); + am.flushAll(0); + EXPECT_TRUE(ia1.load()); + EXPECT_EQUAL(20u, ia1.getBestSnapshot().syncToken); + } +} + +TEST_F("require that extra attribute is added", Fixture) +{ + AttributeVector::SP extra(new Int32Attribute("extra")); + f._m.addExtraAttribute(extra); + AttributeGuard::UP exguard(f._m.getAttribute("extra")); + EXPECT_TRUE(dynamic_cast<Int32Attribute *>(exguard->operator->()) != + NULL); +} + +TEST_F("require that reconfig can add attributes", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP ex(new Int32Attribute("ex")); + f._m.addExtraAttribute(ex); + + AttrSpecList newSpec; + newSpec.push_back(AttrSpec("a1", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a2", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a3", INT32_SINGLE)); + + SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, f._m.getNumDocs(), 0)); + std::vector<AttributeGuard> list; + sam.mgr.getAttributeList(list); + std::sort(list.begin(), list.end(), [](const AttributeGuard & a, const AttributeGuard & b) { + return a->getName() < b->getName(); + }); + EXPECT_EQUAL(3u, list.size()); + EXPECT_EQUAL("a1", list[0]->getName()); + EXPECT_TRUE(list[0].operator->() == a1.get()); // reuse + EXPECT_EQUAL("a2", list[1]->getName()); + EXPECT_EQUAL("a3", list[2]->getName()); + EXPECT_TRUE(sam.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse +} + +TEST_F("require that reconfig can remove attributes", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = f.addAttribute("a2"); + AttributeVector::SP a3 = f.addAttribute("a3"); + + AttrSpecList newSpec; + newSpec.push_back(AttrSpec("a2", INT32_SINGLE)); + + SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 1, 0)); + std::vector<AttributeGuard> list; + sam.mgr.getAttributeList(list); + EXPECT_EQUAL(1u, list.size()); + EXPECT_EQUAL("a2", list[0]->getName()); + EXPECT_TRUE(list[0].operator->() == a2.get()); // reuse +} + +TEST_F("require that new attributes after reconfig are initialized", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + uint32_t docId(0); + a1->addDoc(docId); + EXPECT_EQUAL(1u, docId); + a1->addDoc(docId); + EXPECT_EQUAL(2u, docId); + EXPECT_EQUAL(3u, a1->getNumDocs()); + + AttrSpecList newSpec; + newSpec.push_back(AttrSpec("a1", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a2", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a3", INT32_ARRAY)); + + SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 3, 4)); + AttributeGuard::UP a2ap = sam.mgr.getAttribute("a2"); + AttributeGuard &a2(*a2ap); + EXPECT_EQUAL(3u, a2->getNumDocs()); + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(1))); + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(2))); + EXPECT_EQUAL(0u, a2->getStatus().getLastSyncToken()); + AttributeGuard::UP a3ap = sam.mgr.getAttribute("a3"); + AttributeGuard &a3(*a3ap); + AttributeVector::largeint_t buf[1]; + EXPECT_EQUAL(3u, a3->getNumDocs()); + EXPECT_EQUAL(0u, a3->get(1, buf, 1)); + EXPECT_EQUAL(0u, a3->get(2, buf, 1)); + EXPECT_EQUAL(0u, a3->getStatus().getLastSyncToken()); +} + +TEST_F("require that removed attributes can resurrect", BaseFixture) +{ + proton::AttributeManager::SP am1( + new proton::AttributeManager(test_dir, "test.subdb", + TuneFileAttributes(), + f._fileHeaderContext, + f._attributeFieldWriter)); + { + AttributeVector::SP a1 = + am1->addAttribute("a1", INT32_SINGLE, + 0); + fillAttribute(a1, 2, 10, 15); + EXPECT_EQUAL(3u, a1->getNumDocs()); + } + + AttrSpecList ns1; + SequentialAttributeManager am2(*am1, AttrMgrSpec(ns1, 3, 16)); + am1.reset(); + + AttrSpecList ns2; + ns2.push_back(AttrSpec("a1", INT32_SINGLE)); + // 2 new documents added since a1 was removed + SequentialAttributeManager am3(am2.mgr, AttrMgrSpec(ns2, 5, 20)); + + AttributeGuard::UP ag1ap = am3.mgr.getAttribute("a1"); + AttributeGuard &ag1(*ag1ap); + ASSERT_TRUE(ag1.valid()); + EXPECT_EQUAL(5u, ag1->getNumDocs()); + EXPECT_EQUAL(10, ag1->getInt(1)); + EXPECT_EQUAL(10, ag1->getInt(2)); + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ag1->getInt(3))); + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ag1->getInt(4))); + EXPECT_EQUAL(16u, ag1->getStatus().getLastSyncToken()); +} + +TEST_F("require that extra attribute is not treated as removed", Fixture) +{ + AttributeVector::SP ex(new Int32Attribute("ex")); + f._m.addExtraAttribute(ex); + ex->commit(1,1); + + AttrSpecList ns; + SequentialAttributeManager am2(f._m, AttrMgrSpec(ns, 2, 1)); + EXPECT_TRUE(am2.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse +} + +TEST_F("require that history can be wiped", Fixture) +{ + f.addAttribute("a1"); + f.addAttribute("a2"); + f.addAttribute("a3"); + f._m.flushAll(10); + Schema hs; + hs.addAttributeField(Schema::AttributeField("a1", Schema::INT32)); + hs.addAttributeField(Schema::AttributeField("a3", Schema::INT32)); + f._m.wipeHistory(hs); + FastOS_StatInfo si; + EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a1").c_str(), &si)); + EXPECT_TRUE(FastOS_File::Stat(vespalib::string(test_dir + "/a2").c_str(), &si)); + EXPECT_TRUE(!FastOS_File::Stat(vespalib::string(test_dir + "/a3").c_str(), &si)); +} + +TEST_F("require that lid space can be compacted", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = f.addAttribute("a2"); + AttributeVector::SP ex(new Int32Attribute("ex")); + f._m.addExtraAttribute(ex); + const int64_t attrValue = 33; + fillAttribute(a1, 20, attrValue, 100); + fillAttribute(a2, 20, attrValue, 100); + fillAttribute(ex, 20, attrValue, 100); + + EXPECT_EQUAL(21u, a1->getNumDocs()); + EXPECT_EQUAL(21u, a2->getNumDocs()); + EXPECT_EQUAL(20u, ex->getNumDocs()); + EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit()); + EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit()); + EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); + + f._aw.compactLidSpace(10, 101); + + EXPECT_EQUAL(21u, a1->getNumDocs()); + EXPECT_EQUAL(21u, a2->getNumDocs()); + EXPECT_EQUAL(20u, ex->getNumDocs()); + EXPECT_EQUAL(10u, a1->getCommittedDocIdLimit()); + EXPECT_EQUAL(10u, a2->getCommittedDocIdLimit()); + EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); +} + +TEST_F("require that lid space compaction op can be ignored", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = f.addAttribute("a2"); + AttributeVector::SP ex(new Int32Attribute("ex")); + f._m.addExtraAttribute(ex); + const int64_t attrValue = 33; + fillAttribute(a1, 20, attrValue, 200); + fillAttribute(a2, 20, attrValue, 100); + fillAttribute(ex, 20, attrValue, 100); + + EXPECT_EQUAL(21u, a1->getNumDocs()); + EXPECT_EQUAL(21u, a2->getNumDocs()); + EXPECT_EQUAL(20u, ex->getNumDocs()); + EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit()); + EXPECT_EQUAL(21u, a2->getCommittedDocIdLimit()); + EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); + + f._aw.compactLidSpace(10, 101); + + EXPECT_EQUAL(21u, a1->getNumDocs()); + EXPECT_EQUAL(21u, a2->getNumDocs()); + EXPECT_EQUAL(20u, ex->getNumDocs()); + EXPECT_EQUAL(21u, a1->getCommittedDocIdLimit()); + EXPECT_EQUAL(10u, a2->getCommittedDocIdLimit()); + EXPECT_EQUAL(20u, ex->getCommittedDocIdLimit()); +} + +TEST_F("require that flushed serial number can be retrieved", Fixture) +{ + f.addAttribute("a1"); + EXPECT_EQUAL(0u, f._m.getFlushedSerialNum("a1")); + f._m.flushAll(100); + EXPECT_EQUAL(100u, f._m.getFlushedSerialNum("a1")); + EXPECT_EQUAL(0u, f._m.getFlushedSerialNum("a2")); +} + + +TEST_F("require that writable attributes can be retrieved", Fixture) +{ + auto a1 = f.addAttribute("a1"); + auto a2 = f.addAttribute("a2"); + AttributeVector::SP ex(new Int32Attribute("ex")); + f._m.addExtraAttribute(ex); + auto &vec = f._m.getWritableAttributes(); + EXPECT_EQUAL(2u, vec.size()); + EXPECT_EQUAL(a1.get(), vec[0]); + EXPECT_EQUAL(a2.get(), vec[1]); + EXPECT_EQUAL(a1.get(), f._m.getWritableAttribute("a1")); + EXPECT_EQUAL(a2.get(), f._m.getWritableAttribute("a2")); + AttributeVector *noAttr = nullptr; + EXPECT_EQUAL(noAttr, f._m.getWritableAttribute("a3")); + EXPECT_EQUAL(noAttr, f._m.getWritableAttribute("ex")); +} + + +void +populateAndFlushAttributes(AttributeManagerFixture &f) +{ + const int64_t attrValue = 7; + AttributeVector::SP a1 = f.addAttribute("a1"); + fillAttribute(a1, 1, 10, attrValue, createSerialNum); + AttributeVector::SP a2 = f.addAttribute("a2"); + fillAttribute(a2, 1, 10, attrValue, createSerialNum); + AttributeVector::SP a3 = f.addAttribute("a3"); + fillAttribute(a3, 1, 10, attrValue, createSerialNum); + f._m.flushAll(createSerialNum + 3); +} + +void +validateAttribute(const AttributeVector &attr) +{ + ASSERT_EQUAL(10u, attr.getNumDocs()); + EXPECT_EQUAL(createSerialNum + 3, attr.getStatus().getLastSyncToken()); + for (uint32_t docId = 1; docId < 10; ++docId) { + EXPECT_EQUAL(7, attr.getInt(docId)); + } +} + +TEST_F("require that attributes can be initialized and loaded in sequence", BaseFixture) +{ + { + AttributeManagerFixture amf(f); + populateAndFlushAttributes(amf); + } + { + AttributeManagerFixture amf(f); + + AttrSpecList newSpec; + newSpec.push_back(AttrSpec("a1", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a2", INT32_SINGLE)); + newSpec.push_back(AttrSpec("a3", INT32_SINGLE)); + + SequentialAttributeManager newMgr(amf._m, AttrMgrSpec(newSpec, 10, createSerialNum + 5)); + + AttributeGuard::UP a1 = newMgr.mgr.getAttribute("a1"); + TEST_DO(validateAttribute(a1->get())); + AttributeGuard::UP a2 = newMgr.mgr.getAttribute("a2"); + TEST_DO(validateAttribute(a2->get())); + AttributeGuard::UP a3 = newMgr.mgr.getAttribute("a3"); + TEST_DO(validateAttribute(a3->get())); + } +} + +AttributesConfigBuilder::Attribute +createAttributeConfig(const vespalib::string &name) +{ + AttributesConfigBuilder::Attribute result; + result.name = name; + result.datatype = AttributesConfigBuilder::Attribute::Datatype::INT32; + result.collectiontype = AttributesConfigBuilder::Attribute::Collectiontype::SINGLE; + return result; +} + +TEST_F("require that attributes can be initialized and loaded in parallel", BaseFixture) +{ + { + AttributeManagerFixture amf(f); + populateAndFlushAttributes(amf); + } + { + AttributeManagerFixture amf(f); + + AttributesConfigBuilder attrCfg; + attrCfg.attribute.push_back(createAttributeConfig("a1")); + attrCfg.attribute.push_back(createAttributeConfig("a2")); + attrCfg.attribute.push_back(createAttributeConfig("a3")); + + ParallelAttributeManager newMgr(createSerialNum + 5, amf._msp, attrCfg, 10); + + AttributeGuard::UP a1 = newMgr.mgr->get()->getAttribute("a1"); + TEST_DO(validateAttribute(a1->get())); + AttributeGuard::UP a2 = newMgr.mgr->get()->getAttribute("a2"); + TEST_DO(validateAttribute(a2->get())); + AttributeGuard::UP a3 = newMgr.mgr->get()->getAttribute("a3"); + TEST_DO(validateAttribute(a3->get())); + } +} + +TEST_F("require that we can call functions on all attributes via functor", + Fixture) +{ + f.addAttribute("a1"); + f.addAttribute("a2"); + f.addAttribute("a3"); + std::shared_ptr<MyAttributeFunctor> functor = + std::make_shared<MyAttributeFunctor>(); + f._m.asyncForEachAttribute(functor); + EXPECT_EQUAL("a1,a2,a3", functor->getSortedNames()); +} + +TEST_F("require that we can acquire exclusive read access to attribute", Fixture) +{ + f.addAttribute("attr"); + ExclusiveAttributeReadAccessor::UP attrAccessor = f._m.getExclusiveReadAccessor("attr"); + ExclusiveAttributeReadAccessor::UP noneAccessor = f._m.getExclusiveReadAccessor("none"); + EXPECT_TRUE(attrAccessor.get() != nullptr); + EXPECT_TRUE(noneAccessor.get() == nullptr); +} + +TEST_MAIN() +{ + vespalib::rmdir(test_dir, true); + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore b/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore new file mode 100644 index 00000000000..2400fd559e6 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_populator/.gitignore @@ -0,0 +1 @@ +searchcore_attribute_populator_test_app diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt new file mode 100644 index 00000000000..064759b88d1 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_populator/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_populator_test_app + SOURCES + attribute_populator_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attribute_populator_test_app COMMAND searchcore_attribute_populator_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/DESC b/searchcore/src/tests/proton/attribute/attribute_populator/DESC new file mode 100644 index 00000000000..5ef9dcb2709 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_populator/DESC @@ -0,0 +1 @@ +attribute_populator test. Take a look at attribute_populator_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/FILES b/searchcore/src/tests/proton/attribute/attribute_populator/FILES new file mode 100644 index 00000000000..b6bf0bf8458 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_populator/FILES @@ -0,0 +1 @@ +attribute_populator_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp new file mode 100644 index 00000000000..36e50249b89 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_populator_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/attribute_populator.h> +#include <vespa/searchcore/proton/test/test.h> +#include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> + +using namespace document; +using namespace proton; +using namespace search; +using namespace search::index; + +typedef search::attribute::Config AVConfig; +typedef search::attribute::BasicType AVBasicType; + +const vespalib::string TEST_DIR = "testdir"; +const uint64_t CREATE_SERIAL_NUM = 8u; + +Schema +createSchema() +{ + Schema schema; + schema.addAttributeField(Schema::AttributeField("a1", Schema::DataType::INT32)); + return schema; +} + +struct DocContext +{ + Schema _schema; + DocBuilder _builder; + DocContext() + : _schema(createSchema()), + _builder(_schema) + { + } + Document::UP create(uint32_t id, int64_t fieldValue) { + vespalib::string docId = + vespalib::make_string("id:searchdocument:searchdocument::%u", id); + return _builder.startDocument(docId). + startAttributeField("a1").addInt(fieldValue).endField(). + endDocument(); + } +}; + +struct Fixture +{ + test::DirectoryHandler _testDir; + DummyFileHeaderContext _fileHeader; + ForegroundTaskExecutor _attributeFieldWriter; + AttributeManager::SP _mgr; + AttributePopulator _pop; + DocContext _ctx; + Fixture() + : _testDir(TEST_DIR), + _fileHeader(), + _attributeFieldWriter(), + _mgr(new AttributeManager(TEST_DIR, "test.subdb", + TuneFileAttributes(), + _fileHeader, _attributeFieldWriter)), + _pop(_mgr, 1, "test"), + _ctx() + { + _mgr->addAttribute("a1", AVConfig(AVBasicType::INT32), + CREATE_SERIAL_NUM); + } + AttributeGuard::UP getAttr() { + return _mgr->getAttribute("a1"); + } +}; + +TEST_F("require that reprocess with document populates attribute", Fixture) +{ + AttributeGuard::UP attr = f.getAttr(); + EXPECT_EQUAL(1u, attr->get().getNumDocs()); + + f._pop.handleExisting(5, *f._ctx.create(0, 33)); + EXPECT_EQUAL(6u, attr->get().getNumDocs()); + EXPECT_EQUAL(33, attr->get().getInt(5)); + EXPECT_EQUAL(1u, attr->get().getStatus().getLastSyncToken()); + + f._pop.handleExisting(6, *f._ctx.create(1, 44)); + EXPECT_EQUAL(7u, attr->get().getNumDocs()); + EXPECT_EQUAL(44, attr->get().getInt(6)); + EXPECT_EQUAL(2u, attr->get().getStatus().getLastSyncToken()); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp new file mode 100644 index 00000000000..d5084273c6c --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -0,0 +1,607 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_test"); + +#include <vespa/fastos/file.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/arithmeticvalueupdate.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> +#include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> +#include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/common/idestructorcallback.h> +#include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/util/filekit.h> +#include <vespa/vespalib/io/fileutil.h> + +#include <vespa/document/predicate/predicate_slime_builder.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/predicate_attribute.h> +#include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/searchlib/predicate/predicate_hash.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> +#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/vespalib/tensor/tensor.h> +#include <vespa/vespalib/tensor/types.h> +#include <vespa/vespalib/tensor/default_tensor.h> +#include <vespa/vespalib/tensor/tensor_factory.h> +#include <vespa/searchlib/attribute/tensorattribute.h> + + +namespace vespa { namespace config { namespace search {}}} + +using std::string; +using namespace vespa::config::search; +using namespace config; +using namespace document; +using namespace proton; +using namespace search; +using namespace search::index; +using search::attribute::TensorAttribute; +using search::TuneFileAttributes; +using search::index::DummyFileHeaderContext; +using search::predicate::PredicateIndex; +using search::predicate::PredicateHash; +using vespalib::tensor::Tensor; +using vespalib::tensor::TensorType; +using vespalib::tensor::TensorCells; +using vespalib::tensor::TensorDimensions; + +typedef search::attribute::Config AVConfig; +typedef search::attribute::BasicType AVBasicType; +typedef search::attribute::CollectionType AVCollectionType; +typedef proton::AttributeCollectionSpec::Attribute AttrSpec; +typedef proton::AttributeCollectionSpec::AttributeList AttrSpecList; +typedef proton::AttributeCollectionSpec AttrMgrSpec; +typedef SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > Int32AttributeVector; + +namespace +{ + +const uint64_t createSerialNum = 42u; + +} + +AVConfig +unregister(const AVConfig & cfg) +{ + AVConfig retval = cfg; + return retval; +} + +const string test_dir = "test_output"; +const AVConfig INT32_SINGLE = unregister(AVConfig(AVBasicType::INT32)); +const AVConfig INT32_ARRAY = unregister(AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)); + +void +fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) +{ + test::AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); +} + +void +fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) +{ + test::AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); +} + +const std::shared_ptr<IDestructorCallback> emptyCallback; + + +struct Fixture +{ + test::DirectoryHandler _dirHandler; + DummyFileHeaderContext _fileHeaderContext; + ForegroundTaskExecutor _attributeFieldWriter; + proton::AttributeManager::SP _m; + AttributeWriter aw; + + Fixture() + : _dirHandler(test_dir), + _fileHeaderContext(), + _attributeFieldWriter(), + _m(std::make_shared<proton::AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), + _fileHeaderContext, _attributeFieldWriter)), + aw(_m) + { + } + AttributeVector::SP addAttribute(const vespalib::string &name) { + return _m->addAttribute(name, AVConfig(AVBasicType::INT32), + createSerialNum); + } + void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, + bool immediateCommit = true) { + aw.put(serialNum, doc, lid, immediateCommit, emptyCallback); + } + void update(SerialNum serialNum, const DocumentUpdate &upd, + DocumentIdT lid, bool immediateCommit) { + aw.update(serialNum, upd, lid, immediateCommit, emptyCallback); + } + void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) { + aw.remove(serialNum, lid, immediateCommit, emptyCallback); + } + void commit(SerialNum serialNum) { + aw.commit(serialNum, emptyCallback); + } +}; + + +TEST_F("require that attribute adapter handles put", Fixture) +{ + Schema s; + s.addAttributeField(Schema::AttributeField("a1", Schema::INT32, Schema::SINGLE)); + s.addAttributeField(Schema::AttributeField("a2", Schema::INT32, Schema::ARRAY)); + s.addAttributeField(Schema::AttributeField("a3", Schema::FLOAT, Schema::SINGLE)); + s.addAttributeField(Schema::AttributeField("a4", Schema::STRING, Schema::SINGLE)); + + DocBuilder idb(s); + + proton::AttributeManager & am = *f._m; + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = + am.addAttribute("a2", + AVConfig(AVBasicType::INT32, + AVCollectionType::ARRAY), + createSerialNum); + AttributeVector::SP a3 = + am.addAttribute("a3", AVConfig(AVBasicType::FLOAT), + createSerialNum); + AttributeVector::SP a4 = am.addAttribute("a4", + AVConfig(AVBasicType::STRING), + createSerialNum); + + attribute::IntegerContent ibuf; + attribute::FloatContent fbuf; + attribute::ConstCharContent sbuf; + { // empty document should give default values + EXPECT_EQUAL(1u, a1->getNumDocs()); + f.put(1, *idb.startDocument("doc::1").endDocument(), 1); + EXPECT_EQUAL(2u, a1->getNumDocs()); + EXPECT_EQUAL(2u, a2->getNumDocs()); + EXPECT_EQUAL(2u, a3->getNumDocs()); + EXPECT_EQUAL(2u, a4->getNumDocs()); + EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQUAL(1u, a2->getStatus().getLastSyncToken()); + EXPECT_EQUAL(1u, a3->getStatus().getLastSyncToken()); + EXPECT_EQUAL(1u, a4->getStatus().getLastSyncToken()); + ibuf.fill(*a1, 1); + EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ibuf[0])); + ibuf.fill(*a2, 1); + EXPECT_EQUAL(0u, ibuf.size()); + fbuf.fill(*a3, 1); + EXPECT_EQUAL(1u, fbuf.size()); + EXPECT_TRUE(search::attribute::isUndefined<float>(fbuf[0])); + sbuf.fill(*a4, 1); + EXPECT_EQUAL(1u, sbuf.size()); + EXPECT_EQUAL(strcmp("", sbuf[0]), 0); + } + { // document with single value & multi value attribute + Document::UP doc = idb.startDocument("doc::2"). + startAttributeField("a1").addInt(10).endField(). + startAttributeField("a2").startElement().addInt(20).endElement(). + startElement().addInt(30).endElement().endField().endDocument(); + f.put(2, *doc, 2); + EXPECT_EQUAL(3u, a1->getNumDocs()); + EXPECT_EQUAL(3u, a2->getNumDocs()); + EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQUAL(2u, a2->getStatus().getLastSyncToken()); + EXPECT_EQUAL(2u, a3->getStatus().getLastSyncToken()); + EXPECT_EQUAL(2u, a4->getStatus().getLastSyncToken()); + ibuf.fill(*a1, 2); + EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQUAL(10u, ibuf[0]); + ibuf.fill(*a2, 2); + EXPECT_EQUAL(2u, ibuf.size()); + EXPECT_EQUAL(20u, ibuf[0]); + EXPECT_EQUAL(30u, ibuf[1]); + } + { // replace existing document + Document::UP doc = idb.startDocument("doc::2"). + startAttributeField("a1").addInt(100).endField(). + startAttributeField("a2").startElement().addInt(200).endElement(). + startElement().addInt(300).endElement(). + startElement().addInt(400).endElement().endField().endDocument(); + f.put(3, *doc, 2); + EXPECT_EQUAL(3u, a1->getNumDocs()); + EXPECT_EQUAL(3u, a2->getNumDocs()); + EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQUAL(3u, a2->getStatus().getLastSyncToken()); + EXPECT_EQUAL(3u, a3->getStatus().getLastSyncToken()); + EXPECT_EQUAL(3u, a4->getStatus().getLastSyncToken()); + ibuf.fill(*a1, 2); + EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQUAL(100u, ibuf[0]); + ibuf.fill(*a2, 2); + EXPECT_EQUAL(3u, ibuf.size()); + EXPECT_EQUAL(200u, ibuf[0]); + EXPECT_EQUAL(300u, ibuf[1]); + EXPECT_EQUAL(400u, ibuf[2]); + } +} + +TEST_F("require that attribute adapter handles predicate put", Fixture) +{ + Schema s; + s.addAttributeField( + Schema::AttributeField("a1", Schema::BOOLEANTREE, Schema::SINGLE)); + DocBuilder idb(s); + + proton::AttributeManager & am = *f._m; + AttributeVector::SP a1 = am.addAttribute("a1", + AVConfig(AVBasicType::PREDICATE), + createSerialNum); + + PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); + + // empty document should give default values + EXPECT_EQUAL(1u, a1->getNumDocs()); + f.put(1, *idb.startDocument("doc::1").endDocument(), 1); + EXPECT_EQUAL(2u, a1->getNumDocs()); + EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + + // document with single value attribute + PredicateSlimeBuilder builder; + Document::UP doc = + idb.startDocument("doc::2").startAttributeField("a1") + .addPredicate(builder.true_predicate().build()) + .endField().endDocument(); + f.put(2, *doc, 2); + EXPECT_EQUAL(3u, a1->getNumDocs()); + EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + + auto it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); + EXPECT_FALSE(it.valid()); + + // replace existing document + doc = idb.startDocument("doc::2").startAttributeField("a1") + .addPredicate(builder.feature("foo").value("bar").build()) + .endField().endDocument(); + f.put(3, *doc, 2); + EXPECT_EQUAL(3u, a1->getNumDocs()); + EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + + it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); + EXPECT_TRUE(it.valid()); +} + +TEST_F("require that attribute adapter handles remove", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = f.addAttribute("a2"); + Schema s; + s.addAttributeField(Schema::AttributeField("a1", Schema::INT32, Schema::SINGLE)); + s.addAttributeField(Schema::AttributeField("a2", Schema::INT32, Schema::SINGLE)); + + DocBuilder idb(s); + + fillAttribute(a1, 1, 10, 1); + fillAttribute(a2, 1, 20, 1); + + f.remove(2, 0); + + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a1->getInt(0))); + EXPECT_TRUE(search::attribute::isUndefined<int32_t>(a2->getInt(0))); + + f.remove(2, 0); // same sync token as previous + try { + f.remove(1, 0); // lower sync token than previous + EXPECT_TRUE(true); // update is ignored + } catch (vespalib::IllegalStateException & e) { + LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); + EXPECT_TRUE(true); + } +} + +void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) +{ + attribute::ConstCharContent sbuf; + sbuf.fill(v, lid); + EXPECT_EQUAL(1u, sbuf.size()); + EXPECT_EQUAL(expected, sbuf[0]); +} + +TEST_F("require that visibilitydelay is honoured", Fixture) +{ + proton::AttributeManager & am = *f._m; + AttributeVector::SP a1 = am.addAttribute("a1", + AVConfig(AVBasicType::STRING), + createSerialNum); + Schema s; + s.addAttributeField(Schema::AttributeField("a1", Schema::STRING, Schema::SINGLE)); + DocBuilder idb(s); + EXPECT_EQUAL(1u, a1->getNumDocs()); + EXPECT_EQUAL(0u, a1->getStatus().getLastSyncToken()); + Document::UP doc = idb.startDocument("doc::1") + .startAttributeField("a1").addStr("10").endField() + .endDocument(); + f.put(3, *doc, 1); + EXPECT_EQUAL(2u, a1->getNumDocs()); + EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + AttributeWriter awDelayed(f._m); + awDelayed.put(4, *doc, 2, false, emptyCallback); + EXPECT_EQUAL(3u, a1->getNumDocs()); + EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + awDelayed.put(5, *doc, 4, false, emptyCallback); + EXPECT_EQUAL(5u, a1->getNumDocs()); + EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + awDelayed.commit(6, emptyCallback); + EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + + AttributeWriter awDelayedShort(f._m); + awDelayedShort.put(7, *doc, 2, false, emptyCallback); + EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + awDelayedShort.put(8, *doc, 2, false, emptyCallback); + awDelayedShort.commit(8, emptyCallback); + EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + + verifyAttributeContent(*a1, 2, "10"); + awDelayed.put(9, *idb.startDocument("doc::1").startAttributeField("a1").addStr("11").endField().endDocument(), + 2, false, emptyCallback); + awDelayed.put(10, *idb.startDocument("doc::1").startAttributeField("a1").addStr("20").endField().endDocument(), + 2, false, emptyCallback); + awDelayed.put(11, *idb.startDocument("doc::1").startAttributeField("a1").addStr("30").endField().endDocument(), + 2, false, emptyCallback); + EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + verifyAttributeContent(*a1, 2, "10"); + awDelayed.commit(12, emptyCallback); + EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken()); + verifyAttributeContent(*a1, 2, "30"); + +} + +TEST_F("require that attribute adapter handles predicate remove", Fixture) +{ + proton::AttributeManager & am = *f._m; + AttributeVector::SP a1 = am.addAttribute("a1", + AVConfig(AVBasicType::PREDICATE), + createSerialNum); + Schema s; + s.addAttributeField( + Schema::AttributeField("a1", Schema::BOOLEANTREE, Schema::SINGLE)); + + DocBuilder idb(s); + PredicateSlimeBuilder builder; + Document::UP doc = + idb.startDocument("doc::1").startAttributeField("a1") + .addPredicate(builder.true_predicate().build()) + .endField().endDocument(); + f.put(1, *doc, 1); + EXPECT_EQUAL(2u, a1->getNumDocs()); + + PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); + EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + f.remove(2, 1); + EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); +} + +TEST_F("require that attribute adapter handles update", Fixture) +{ + AttributeVector::SP a1 = f.addAttribute("a1"); + AttributeVector::SP a2 = f.addAttribute("a2"); + + fillAttribute(a1, 1, 10, 1); + fillAttribute(a2, 1, 20, 1); + + Schema schema; + schema.addAttributeField(Schema::AttributeField( + "a1", Schema::INT32, + Schema::SINGLE)); + schema.addAttributeField(Schema::AttributeField( + "a2", Schema::INT32, + Schema::SINGLE)); + DocBuilder idb(schema); + const document::DocumentType &dt(idb.getDocumentType()); + DocumentUpdate upd(dt, DocumentId("doc::1")); + upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) + .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 5))); + upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) + .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10))); + + bool immediateCommit = true; + f.update(2, upd, 1, immediateCommit); + + attribute::IntegerContent ibuf; + ibuf.fill(*a1, 1); + EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQUAL(15u, ibuf[0]); + ibuf.fill(*a2, 1); + EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQUAL(30u, ibuf[0]); + + f.update(2, upd, 1, immediateCommit); // same sync token as previous + try { + f.update(1, upd, 1, immediateCommit); // lower sync token than previous + EXPECT_TRUE(true); // update is ignored + } catch (vespalib::IllegalStateException & e) { + LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); + EXPECT_TRUE(true); + } +} + +TEST_F("require that attribute adapter handles predicate update", Fixture) +{ + proton::AttributeManager & am = *f._m; + AttributeVector::SP a1 = am.addAttribute("a1", + AVConfig(AVBasicType::PREDICATE), + createSerialNum); + Schema schema; + schema.addAttributeField(Schema::AttributeField( + "a1", Schema::BOOLEANTREE, + Schema::SINGLE)); + + DocBuilder idb(schema); + PredicateSlimeBuilder builder; + Document::UP doc = + idb.startDocument("doc::1").startAttributeField("a1") + .addPredicate(builder.true_predicate().build()) + .endField().endDocument(); + f.put(1, *doc, 1); + EXPECT_EQUAL(2u, a1->getNumDocs()); + + const document::DocumentType &dt(idb.getDocumentType()); + DocumentUpdate upd(dt, DocumentId("doc::1")); + PredicateFieldValue new_value(builder.feature("foo").value("bar").build()); + upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) + .addUpdate(AssignValueUpdate(new_value))); + + PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); + EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); + bool immediateCommit = true; + f.update(2, upd, 1, immediateCommit); + EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); +} + +struct AttributeCollectionSpecFixture +{ + AttributesConfigBuilder _builder; + AttributeCollectionSpecFactory _factory; + AttributeCollectionSpecFixture(bool fastAccessOnly) + : _builder(), + _factory(search::GrowStrategy(), 100, fastAccessOnly) + { + addAttribute("a1", false); + addAttribute("a2", true); + } + void addAttribute(const vespalib::string &name, bool fastAccess) { + AttributesConfigBuilder::Attribute attr; + attr.name = name; + attr.fastaccess = fastAccess; + _builder.attribute.push_back(attr); + } + AttributeCollectionSpec::UP create(uint32_t docIdLimit, + search::SerialNum serialNum) { + return _factory.create(_builder, docIdLimit, serialNum); + } +}; + +struct NormalAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture +{ + NormalAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(false) {} +}; + +struct FastAccessAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture +{ + FastAccessAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(true) {} +}; + +TEST_F("require that normal attribute collection spec can be created", + NormalAttributeCollectionSpecFixture) +{ + AttributeCollectionSpec::UP spec = f.create(10, 20); + EXPECT_EQUAL(2u, spec->getAttributes().size()); + EXPECT_EQUAL("a1", spec->getAttributes()[0].getName()); + EXPECT_EQUAL("a2", spec->getAttributes()[1].getName()); + EXPECT_EQUAL(10u, spec->getDocIdLimit()); + EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); +} + +TEST_F("require that fast access attribute collection spec can be created", + FastAccessAttributeCollectionSpecFixture) +{ + AttributeCollectionSpec::UP spec = f.create(10, 20); + EXPECT_EQUAL(1u, spec->getAttributes().size()); + EXPECT_EQUAL("a2", spec->getAttributes()[0].getName()); + EXPECT_EQUAL(10u, spec->getDocIdLimit()); + EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); +} + +const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"}; + +struct FilterFixture +{ + test::DirectoryHandler _dirHandler; + DummyFileHeaderContext _fileHeaderContext; + ForegroundTaskExecutor _attributeFieldWriter; + proton::AttributeManager::SP _baseMgr; + FilterAttributeManager _filterMgr; + FilterFixture() + : _dirHandler(test_dir), + _fileHeaderContext(), + _attributeFieldWriter(), + _baseMgr(new proton::AttributeManager(test_dir, "test.subdb", + TuneFileAttributes(), + _fileHeaderContext, + _attributeFieldWriter)), + _filterMgr(ACCEPTED_ATTRIBUTES, _baseMgr) + { + _baseMgr->addAttribute("a1", INT32_SINGLE, createSerialNum); + _baseMgr->addAttribute("a2", INT32_SINGLE, createSerialNum); + } +}; + +TEST_F("require that filter attribute manager can filter attributes", FilterFixture) +{ + EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == NULL); + EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != NULL); + std::vector<AttributeGuard> attrs; + f._filterMgr.getAttributeList(attrs); + EXPECT_EQUAL(1u, attrs.size()); + EXPECT_EQUAL("a2", attrs[0].get().getName()); +} + +TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture) +{ + f._baseMgr->flushAll(100); + EXPECT_EQUAL(0u, f._filterMgr.getFlushedSerialNum("a1")); + EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2")); +} + +namespace { + +Tensor::UP +createTensor(const TensorCells &cells, const TensorDimensions &dimensions) { + vespalib::tensor::DefaultTensor::builder builder; + return vespalib::tensor::TensorFactory::create(cells, dimensions, builder); +} + +} + + +TEST_F("Test that we can use attribute writer to write to tensor attribute", + Fixture) +{ + proton::AttributeManager & am = *f._m; + AVConfig cfg(AVBasicType::TENSOR); + cfg.setTensorType(TensorType::fromSpec("tensor(x{},y{})")); + AttributeVector::SP a1 = am.addAttribute("a1", + cfg, + createSerialNum); + Schema s; + s.addAttributeField(Schema::AttributeField("a1", Schema::TENSOR, + Schema::SINGLE)); + DocBuilder builder(s); + auto tensor = createTensor({ {{{"x", "4"}, {"y", "5"}}, 7} }, + {"x", "y"}); + Document::UP doc = builder.startDocument("doc::1"). + startAttributeField("a1"). + addTensor(tensor->clone()).endField().endDocument(); + f.put(1, *doc, 1); + EXPECT_EQUAL(2u, a1->getNumDocs()); + TensorAttribute *tensorAttribute = + dynamic_cast<TensorAttribute *>(a1.get()); + EXPECT_TRUE(tensorAttribute != nullptr); + auto tensor2 = tensorAttribute->getTensor(1); + EXPECT_TRUE(static_cast<bool>(tensor2)); + EXPECT_TRUE(tensor->equals(*tensor2)); +} + +TEST_MAIN() +{ + vespalib::rmdir(test_dir, true); + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/attribute_test.sh b/searchcore/src/tests/proton/attribute/attribute_test.sh new file mode 100755 index 00000000000..950a9f92bb8 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +rm -rf test_output +$VALGRIND ./searchcore_attribute_test_app diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore b/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore new file mode 100644 index 00000000000..2642c637ea0 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/.gitignore @@ -0,0 +1 @@ +searchcore_attribute_usage_filter_test_app diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt new file mode 100644 index 00000000000..2dd66c2a3ec --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_usage_filter_test_app + SOURCES + attribute_usage_filter_test.cpp + DEPENDS + searchcore_attribute +) +vespa_add_test(NAME searchcore_attribute_usage_filter_test_app COMMAND searchcore_attribute_usage_filter_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC b/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC new file mode 100644 index 00000000000..31b3afbcdf7 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/DESC @@ -0,0 +1 @@ +AttributeUsageFilter test. Take a look at attribute_usage_filter_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES b/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES new file mode 100644 index 00000000000..b63aeb79d02 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/FILES @@ -0,0 +1 @@ +attribute_usage_filter_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp new file mode 100644 index 00000000000..d8ede8030e2 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_usage_filter_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> + +using proton::AttributeUsageFilter; +using proton::AttributeUsageStats; + +namespace +{ + +search::AddressSpace enumStoreOverLoad(30 * 1024 * 1024 * UINT64_C(1024), + 32 * 1024 * 1024 * UINT64_C(1024)); + +search::AddressSpace multiValueOverLoad(127 * 1024 * 1024, + 128 * 1024 * 1024); + + + +class MyAttributeStats : public AttributeUsageStats +{ +public: + void triggerEnumStoreLimit() { + merge({ enumStoreOverLoad, + search::AddressSpaceUsage::defaultMultiValueUsage() }, + "enumeratedName", + "ready"); + } + + void triggerMultiValueLimit() { + merge({ search::AddressSpaceUsage::defaultEnumStoreUsage(), + multiValueOverLoad }, + "multiValueName", + "ready"); + } +}; + +struct Fixture +{ + AttributeUsageFilter _filter; + using State = AttributeUsageFilter::State; + using Config = AttributeUsageFilter::Config; + + Fixture() + : _filter() + { + } + + void testWrite(const vespalib::string &exp) { + if (exp.empty()) { + EXPECT_TRUE(_filter.acceptWriteOperation()); + State state = _filter.getAcceptState(); + EXPECT_TRUE(state.acceptWriteOperation()); + EXPECT_EQUAL(exp, state.message()); + } else { + EXPECT_FALSE(_filter.acceptWriteOperation()); + State state = _filter.getAcceptState(); + EXPECT_FALSE(state.acceptWriteOperation()); + EXPECT_EQUAL(exp, state.message()); + } + } + + void setAttributeStats(const AttributeUsageStats &stats) { + _filter.setAttributeStats(stats); + } +}; + +} + +TEST_F("Check that default filter allows write", Fixture) +{ + f.testWrite(""); +} + + +TEST_F("Check that enum store limit can be reached", Fixture) +{ + f._filter.setConfig(Fixture::Config(0.8, 1.0)); + MyAttributeStats stats; + stats.triggerEnumStoreLimit(); + f.setAttributeStats(stats); + f.testWrite("enumStoreLimitReached: { " + "action: \"" + "add more content nodes" + "\", " + "reason: \"" + "enum store address space used (0.9375) > limit (0.8)" + "\", " + "enumStore: { used: 32212254720, limit: 34359738368}, " + "attributeName: \"enumeratedName\", subdb: \"ready\"}"); +} + +TEST_F("Check that multivalue limit can be reached", Fixture) +{ + f._filter.setConfig(Fixture::Config(1.0, 0.8)); + MyAttributeStats stats; + stats.triggerMultiValueLimit(); + f.setAttributeStats(stats); + f.testWrite("multiValueLimitReached: { " + "action: \"" + "use 'huge' setting on attribute field " + "or add more content nodes" + "\", " + "reason: \"" + "multiValue address space used (0.992188) > limit (0.8)" + "\", " + "multiValue: { used: 133169152, limit: 134217728}, " + "attributeName: \"multiValueName\", subdb: \"ready\"}"); +} + +TEST_F("Check that both enumstore limit and multivalue limit can be reached", + Fixture) +{ + f._filter.setConfig(Fixture::Config(0.8, 0.8)); + MyAttributeStats stats; + stats.triggerEnumStoreLimit(); + stats.triggerMultiValueLimit(); + f.setAttributeStats(stats); + f.testWrite("enumStoreLimitReached: { " + "action: \"" + "add more content nodes" + "\", " + "reason: \"" + "enum store address space used (0.9375) > limit (0.8)" + "\", " + "enumStore: { used: 32212254720, limit: 34359738368}, " + "attributeName: \"enumeratedName\", subdb: \"ready\"}" + ", " + "multiValueLimitReached: { " + "action: \"" + "use 'huge' setting on attribute field " + "or add more content nodes" + "\", " + "reason: \"" + "multiValue address space used (0.992188) > limit (0.8)" + "\", " + "multiValue: { used: 133169152, limit: 134217728}, " + "attributeName: \"multiValueName\", subdb: \"ready\"}"); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp new file mode 100644 index 00000000000..53904e14658 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -0,0 +1,564 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attributeflush_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/flushableattribute.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/common/indexmetainfo.h> +#include <vespa/searchlib/util/dirtraverse.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> +#include <vespa/searchcore/proton/test/directory_handler.h> + +#include <vespa/searchlib/attribute/attributevector.hpp> + +using namespace document; +using namespace search; +using namespace vespalib; + +using search::index::DummyFileHeaderContext; + +typedef search::attribute::Config AVConfig; +typedef search::attribute::BasicType AVBasicType; +typedef search::attribute::CollectionType AVCollectionType; + +typedef std::shared_ptr<Gate> GateSP; + +namespace proton { + +namespace +{ + +const uint64_t createSerialNum = 42u; + +} + +class TaskWrapper : public Executor::Task +{ +private: + Executor::Task::UP _task; + GateSP _gate; +public: + TaskWrapper(Executor::Task::UP task, const GateSP &gate) + : _task(std::move(task)), + _gate(gate) + { + } + + virtual void + run(void) + { + _task->run(); + _gate->countDown(); + LOG(info, "doneFlushing"); + } +}; + + +class FlushHandler +{ +private: + ThreadStackExecutor _executor; +public: + GateSP gate; + + FlushHandler() + : _executor(1, 65536), + gate() + { + } + + void + doFlushing(Executor::Task::UP task) + { + Executor::Task::UP wrapper(new TaskWrapper(std::move(task), gate)); + Executor::Task::UP ok = _executor.execute(std::move(wrapper)); + assert(ok.get() == NULL); + } +}; + + +class UpdaterTask +{ +private: + proton::AttributeManager & _am; +public: + UpdaterTask(proton::AttributeManager & am) + : + _am(am) + { + } + + void + startFlushing(uint64_t syncToken, FlushHandler & handler); + + void + run(void); +}; + + +void +UpdaterTask::startFlushing(uint64_t syncToken, FlushHandler & handler) +{ + handler.gate.reset(new Gate()); + IFlushTarget::SP flushable = _am.getFlushable("a1"); + LOG(info, "startFlushing(%" PRIu64 ")", syncToken); + handler.doFlushing(flushable->initFlush(syncToken)); +} + + +void +UpdaterTask::run(void) +{ + LOG(info, "UpdaterTask::run(begin)"); + uint32_t totalDocs = 2000000; + uint32_t totalDocsMax = 125000000; // XXX: Timing dependent. + uint32_t slowdownUpdateLim = 4000000; + bool slowedDown = false; + uint32_t incDocs = 1000; + uint64_t commits = 0; + uint32_t flushCount = 0; + uint64_t flushedToken = 0; + uint64_t needFlushToken = 0; + FlushHandler flushHandler; + for (uint32_t i = incDocs; + i <= totalDocs || (flushCount + (flushedToken < + needFlushToken) <= 2 && + i <= totalDocsMax); + i += incDocs) { + uint32_t startDoc = 0; + uint32_t lastDoc = 0; + AttributeGuard::UP agap = _am.getAttribute("a1"); + AttributeGuard &ag(*agap); + IntegerAttribute & ia = static_cast<IntegerAttribute &>(*ag); + for (uint32_t j = i - incDocs; j < i; ++j) { + if (j >= ag->getNumDocs()) { + ag->addDocs(startDoc, lastDoc, incDocs); + if (i % (totalDocs / 20) == 0) { + LOG(info, + "addDocs(%u, %u, %u)", + startDoc, lastDoc, ag->getNumDocs()); + } + } + ia.update(j, i); + } + ia.commit(i-1, i); // save i as last sync token + needFlushToken = i; + assert(i + 1 == ag->getNumDocs()); + if ((commits++ % 20 == 0) && + (flushHandler.gate.get() == NULL || + flushHandler.gate->getCount() == 0)) { + startFlushing(i, flushHandler); + ++flushCount; + flushedToken = i; + slowedDown = false; + } + if (needFlushToken > flushedToken + slowdownUpdateLim) { + FastOS_Thread::Sleep(100); + if (!slowedDown) { + LOG(warning, + "Slowing down updates due to slow flushing (slow disk ?)"); + } + slowedDown = true; + } + } + if (flushHandler.gate.get() != NULL) { + flushHandler.gate->await(); + } + if (flushedToken < needFlushToken) { + startFlushing(needFlushToken, flushHandler); + flushHandler.gate->await(); + } + LOG(info, "UpdaterTask::run(end)"); +} + + +AVConfig +getInt32Config() +{ + return AVConfig(AVBasicType::INT32); +} + + +class Test : public vespalib::TestApp +{ +private: + void + requireThatUpdaterAndFlusherCanRunConcurrently(void); + + void + requireThatFlushableAttributeReportsMemoryUsage(void); + + void + requireThatFlushableAttributeManagesSyncTokenInfo(void); + + void + requireThatFlushTargetsCanBeRetrieved(void); + + void + requireThatCleanUpIsPerformedAfterFlush(void); + + void + requireThatFlushStatsAreUpdated(void); + + void + requireThatOnlyOneFlusherCanRunAtTheSameTime(void); + + void + requireThatLastFlushTimeIsReported(void); + + void + requireThatShrinkWorks(); +public: + int + Main(void); +}; + + +const string test_dir = "flush"; + +struct BaseFixture +{ + test::DirectoryHandler _dirHandler; + DummyFileHeaderContext _fileHeaderContext; + ForegroundTaskExecutor _attributeFieldWriter; + BaseFixture() + : _dirHandler(test_dir), + _fileHeaderContext(), + _attributeFieldWriter() + { + } +}; + + +struct AttributeManagerFixture +{ + AttributeManager::SP _msp; + AttributeManager &_m; + AttributeWriter _aw; + AttributeManagerFixture(BaseFixture &bf) + : _msp(std::make_shared<AttributeManager> + (test_dir, "test.subdb", TuneFileAttributes(), bf._fileHeaderContext, + bf._attributeFieldWriter)), + _m(*_msp), + _aw(_msp) + { + } + AttributeVector::SP addAttribute(const vespalib::string &name) { + return _m.addAttribute(name, getInt32Config(), createSerialNum); + } +}; + +struct Fixture : public BaseFixture, public AttributeManagerFixture +{ + Fixture() + : BaseFixture(), + AttributeManagerFixture(*static_cast<BaseFixture *>(this)) + { + } +}; + + + +void +Test::requireThatUpdaterAndFlusherCanRunConcurrently(void) +{ + Fixture f; + AttributeManager &am = f._m; + EXPECT_TRUE(f.addAttribute("a1").get() != NULL); + IFlushTarget::SP ft = am.getFlushable("a1"); + (static_cast<FlushableAttribute *>(ft.get()))->setCleanUpAfterFlush(false); + UpdaterTask updaterTask(am); + updaterTask.run(); + + IndexMetaInfo info("flush/a1"); + EXPECT_TRUE(info.load()); + EXPECT_TRUE(info.snapshots().size() > 2); + for (size_t i = 0; i < info.snapshots().size(); ++i) { + const IndexMetaInfo::Snapshot & snap = info.snapshots()[i]; + LOG(info, + "Snapshot(%" PRIu64 ", %s)", + snap.syncToken, snap.dirName.c_str()); + if (snap.syncToken > 0) { + EXPECT_TRUE(snap.valid); + std::string baseFileName = "flush/a1/" + snap.dirName + "/a1"; + AttributeVector::SP attr = + AttributeFactory::createAttribute(baseFileName, + getInt32Config()); + EXPECT_TRUE(attr->load()); + EXPECT_EQUAL((uint32_t)snap.syncToken + 1, attr->getNumDocs()); + } + } +} + + +void +Test::requireThatFlushableAttributeReportsMemoryUsage(void) +{ + Fixture f; + AttributeManager &am = f._m; + AttributeVector::SP av = f.addAttribute("a2"); + av->addDocs(100); + av->commit(); + IFlushTarget::SP fa = am.getFlushable("a2"); + EXPECT_TRUE(av->getStatus().getAllocated() >= 100u * sizeof(int32_t)); + EXPECT_EQUAL(av->getStatus().getUsed(), + fa->getApproxMemoryGain().getBefore()+0lu); + // attributes stay in memory + EXPECT_EQUAL(fa->getApproxMemoryGain().getBefore(), + fa->getApproxMemoryGain().getAfter()); +} + + +void +Test::requireThatFlushableAttributeManagesSyncTokenInfo(void) +{ + Fixture f; + AttributeManager &am = f._m; + AttributeVector::SP av = f.addAttribute("a3"); + av->addDocs(1); + IFlushTarget::SP fa = am.getFlushable("a3"); + + IndexMetaInfo info("flush/a3"); + EXPECT_EQUAL(0u, fa->getFlushedSerialNum()); + EXPECT_TRUE(fa->initFlush(0).get() == NULL); + EXPECT_TRUE(info.load()); + EXPECT_EQUAL(0u, info.snapshots().size()); + + av->commit(10, 10); // last sync token = 10 + EXPECT_EQUAL(0u, fa->getFlushedSerialNum()); + EXPECT_TRUE(fa->initFlush(10).get() != NULL); + fa->initFlush(10)->run(); + EXPECT_EQUAL(10u, fa->getFlushedSerialNum()); + EXPECT_TRUE(info.load()); + EXPECT_EQUAL(1u, info.snapshots().size()); + EXPECT_TRUE(info.snapshots()[0].valid); + EXPECT_EQUAL(10u, info.snapshots()[0].syncToken); + + av->commit(20, 20); // last sync token = 20 + EXPECT_EQUAL(10u, fa->getFlushedSerialNum()); + fa->initFlush(20)->run(); + EXPECT_EQUAL(20u, fa->getFlushedSerialNum()); + EXPECT_TRUE(info.load()); + EXPECT_EQUAL(1u, info.snapshots().size()); // snapshot 10 removed + EXPECT_TRUE(info.snapshots()[0].valid); + EXPECT_EQUAL(20u, info.snapshots()[0].syncToken); +} + + +void +Test::requireThatFlushTargetsCanBeRetrieved(void) +{ + Fixture f; + AttributeManager &am = f._m; + f.addAttribute("a4"); + f.addAttribute("a5"); + std::vector<IFlushTarget::SP> ftl = am.getFlushTargets(); + EXPECT_EQUAL(2u, ftl.size()); + EXPECT_EQUAL(am.getFlushable("a4").get(), ftl[0].get()); + EXPECT_EQUAL(am.getFlushable("a5").get(), ftl[1].get()); +} + + +void +Test::requireThatCleanUpIsPerformedAfterFlush(void) +{ + Fixture f; + AttributeVector::SP av = f.addAttribute("a6"); + av->addDocs(1); + av->commit(30, 30); + + // fake up some snapshots + std::string snap10 = "flush/a6/snapshot-10"; + std::string snap20 = "flush/a6/snapshot-20"; + vespalib::mkdir(snap10, false); + vespalib::mkdir(snap20, false); + IndexMetaInfo info("flush/a6"); + info.addSnapshot(IndexMetaInfo::Snapshot(true, 10, "snapshot-10")); + info.addSnapshot(IndexMetaInfo::Snapshot(false, 20, "snapshot-20")); + EXPECT_TRUE(info.save()); + + FlushableAttribute fa(av, "flush", TuneFileAttributes(), + f._fileHeaderContext, f._attributeFieldWriter); + fa.initFlush(30)->run(); + + EXPECT_TRUE(info.load()); + EXPECT_EQUAL(1u, info.snapshots().size()); // snapshots 10 & 20 removed + EXPECT_TRUE(info.snapshots()[0].valid); + EXPECT_EQUAL(30u, info.snapshots()[0].syncToken); + FastOS_StatInfo statInfo; + EXPECT_TRUE(!FastOS_File::Stat(snap10.c_str(), &statInfo)); + EXPECT_TRUE(!FastOS_File::Stat(snap20.c_str(), &statInfo)); +} + + +void +Test::requireThatFlushStatsAreUpdated(void) +{ + Fixture f; + AttributeManager &am = f._m; + AttributeVector::SP av = f.addAttribute("a7"); + av->addDocs(1); + av->commit(100,100); + IFlushTarget::SP ft = am.getFlushable("a7"); + ft->initFlush(101)->run(); + FlushStats stats = ft->getLastFlushStats(); + EXPECT_EQUAL("flush/a7/snapshot-101", stats.getPath()); + EXPECT_EQUAL(8u, stats.getPathElementsToLog()); +} + + +void +Test::requireThatOnlyOneFlusherCanRunAtTheSameTime(void) +{ + Fixture f; + AttributeManager &am = f._m; + AttributeVector::SP av = f.addAttribute("a8"); + av->addDocs(10000); + av->commit(9,9); + IFlushTarget::SP ft = am.getFlushable("a8"); + (static_cast<FlushableAttribute *>(ft.get()))->setCleanUpAfterFlush(false); + vespalib::ThreadStackExecutor exec(16, 64000); + + for (size_t i = 10; i < 100; ++i) { + av->commit(i, i); + vespalib::Executor::Task::UP task = ft->initFlush(i); + exec.execute(std::move(task)); + } + exec.sync(); + exec.shutdown(); + + IndexMetaInfo info("flush/a8"); + ASSERT_TRUE(info.load()); + LOG(info, "Found %zu snapshots", info.snapshots().size()); + for (size_t i = 0; i < info.snapshots().size(); ++i) { + EXPECT_EQUAL(true, info.snapshots()[i].valid); + } + IndexMetaInfo::Snapshot best = info.getBestSnapshot(); + EXPECT_EQUAL(true, best.valid); + EXPECT_EQUAL(99u, best.syncToken); + FlushStats stats = ft->getLastFlushStats(); + EXPECT_EQUAL("flush/a8/snapshot-99", stats.getPath()); +} + + +void +Test::requireThatLastFlushTimeIsReported(void) +{ + BaseFixture f; + FastOS_StatInfo stat; + { // no meta info file yet + AttributeManagerFixture amf(f); + AttributeManager &am = amf._m; + AttributeVector::SP av = amf.addAttribute("a9"); + EXPECT_EQUAL(0, am.getFlushable("a9")->getLastFlushTime().time()); + } + { // no snapshot flushed yet + AttributeManagerFixture amf(f); + AttributeManager &am = amf._m; + AttributeVector::SP av = amf.addAttribute("a9"); + IFlushTarget::SP ft = am.getFlushable("a9"); + EXPECT_EQUAL(0, ft->getLastFlushTime().time()); + ft->initFlush(5)->run(); + EXPECT_TRUE(FastOS_File::Stat("flush/a9/snapshot-5", &stat)); + EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time()); + } + { // snapshot flushed + AttributeManagerFixture amf(f); + AttributeManager &am = amf._m; + amf.addAttribute("a9"); + IFlushTarget::SP ft = am.getFlushable("a9"); + EXPECT_EQUAL(stat._modifiedTime, ft->getLastFlushTime().time()); + { // updated flush time after nothing to flush + FastOS_Thread::Sleep(8000); + fastos::TimeStamp now = fastos::ClockSystem::now(); + Executor::Task::UP task = ft->initFlush(5); + EXPECT_TRUE(task.get() == NULL); + EXPECT_LESS(stat._modifiedTime, ft->getLastFlushTime().time()); + EXPECT_APPROX(now.time(), ft->getLastFlushTime().time(), 8); + } + } +} + + +void +Test::requireThatShrinkWorks() +{ + Fixture f; + AttributeManager &am = f._m; + AttributeVector::SP av = f.addAttribute("a10"); + + av->addDocs(1000 - av->getNumDocs()); + av->commit(10, 10); + IFlushTarget::SP ft = am.getFlushable("a10"); + EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), + ft->getApproxMemoryGain().getAfter()); + AttributeGuard::UP g = am.getAttribute("a10"); + EXPECT_FALSE(av->wantShrinkLidSpace()); + EXPECT_FALSE(av->canShrinkLidSpace()); + EXPECT_EQUAL(1000u, av->getNumDocs()); + EXPECT_EQUAL(1000u, av->getCommittedDocIdLimit()); + av->compactLidSpace(100); + EXPECT_TRUE(av->wantShrinkLidSpace()); + EXPECT_FALSE(av->canShrinkLidSpace()); + EXPECT_EQUAL(1000u, av->getNumDocs()); + EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); + f._aw.heartBeat(11); + EXPECT_TRUE(av->wantShrinkLidSpace()); + EXPECT_FALSE(av->canShrinkLidSpace()); + EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), + ft->getApproxMemoryGain().getAfter()); + g.reset(); + f._aw.heartBeat(11); + EXPECT_TRUE(av->wantShrinkLidSpace()); + EXPECT_TRUE(av->canShrinkLidSpace()); + EXPECT_TRUE(ft->getApproxMemoryGain().getBefore() > + ft->getApproxMemoryGain().getAfter()); + EXPECT_EQUAL(1000u, av->getNumDocs()); + EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); + vespalib::ThreadStackExecutor exec(1, 128 * 1024); + vespalib::Executor::Task::UP task = ft->initFlush(11); + exec.execute(std::move(task)); + exec.sync(); + exec.shutdown(); + EXPECT_FALSE(av->wantShrinkLidSpace()); + EXPECT_FALSE(av->canShrinkLidSpace()); + EXPECT_EQUAL(ft->getApproxMemoryGain().getBefore(), + ft->getApproxMemoryGain().getAfter()); + EXPECT_EQUAL(100u, av->getNumDocs()); + EXPECT_EQUAL(100u, av->getCommittedDocIdLimit()); +} + + +int +Test::Main(void) +{ + TEST_INIT("attributeflush_test"); + + if (_argc > 0) { + DummyFileHeaderContext::setCreator(_argv[0]); + } + vespalib::rmdir(test_dir, true); + TEST_DO(requireThatUpdaterAndFlusherCanRunConcurrently()); + TEST_DO(requireThatFlushableAttributeReportsMemoryUsage()); + TEST_DO(requireThatFlushableAttributeManagesSyncTokenInfo()); + TEST_DO(requireThatFlushTargetsCanBeRetrieved()); + TEST_DO(requireThatCleanUpIsPerformedAfterFlush()); + TEST_DO(requireThatFlushStatsAreUpdated()); + TEST_DO(requireThatOnlyOneFlusherCanRunAtTheSameTime()); + TEST_DO(requireThatLastFlushTimeIsReported()); + TEST_DO(requireThatShrinkWorks()); + + TEST_DONE(); +} + +} + +TEST_APPHOOK(proton::Test); diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.sh b/searchcore/src/tests/proton/attribute/attributeflush_test.sh new file mode 100755 index 00000000000..8ec2f5d8dd8 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +rm -rf flush +$VALGRIND ./searchcore_attributeflush_test_app diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore b/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore new file mode 100644 index 00000000000..3b612102a10 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/.gitignore @@ -0,0 +1 @@ +searchcore_attributes_state_explorer_test_app diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt new file mode 100644 index 00000000000..322d22c8f0d --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attributes_state_explorer_test_app + SOURCES + attributes_state_explorer_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_attributes_state_explorer_test_app COMMAND searchcore_attributes_state_explorer_test_app) diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC b/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC new file mode 100644 index 00000000000..1459d32ddae --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/DESC @@ -0,0 +1 @@ +attributes_state_explorer test. Take a look at attributes_state_explorer_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES b/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES new file mode 100644 index 00000000000..f49eb2b8e86 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/FILES @@ -0,0 +1 @@ +attributes_state_explorer_test.cpp diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp new file mode 100644 index 00000000000..43eeec6086a --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attributes_state_explorer_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h> +#include <vespa/searchcore/proton/attribute/attributemanager.h> +#include <vespa/searchcore/proton/test/attribute_vectors.h> +#include <vespa/searchcore/proton/test/directory_handler.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/common/foregroundtaskexecutor.h> +#include <vespa/vespalib/test/insertion_operators.h> + +using namespace proton; +using namespace proton::test; +using search::index::DummyFileHeaderContext; +using search::AttributeVector; +using search::TuneFileAttributes; +using search::ForegroundTaskExecutor; + +const vespalib::string TEST_DIR = "test_output"; + +struct Fixture +{ + DirectoryHandler _dirHandler; + DummyFileHeaderContext _fileHeaderContext; + ForegroundTaskExecutor _attributeFieldWriter; + AttributeManager::SP _mgr; + AttributeManagerExplorer _explorer; + Fixture() + : _dirHandler(TEST_DIR), + _fileHeaderContext(), + _attributeFieldWriter(), + _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), + _fileHeaderContext, + _attributeFieldWriter)), + _explorer(_mgr) + { + addAttribute("regular"); + addExtraAttribute("extra"); + } + void addAttribute(const vespalib::string &name) { + _mgr->addAttribute(name, AttributeUtils::getInt32Config(), 1); + } + void addExtraAttribute(const vespalib::string &name) { + _mgr->addExtraAttribute(AttributeVector::SP(new Int32Attribute(name))); + } +}; + +typedef std::vector<vespalib::string> StringVector; + +TEST_F("require that attributes are exposed as children names", Fixture) +{ + StringVector children = f._explorer.get_children_names(); + std::sort(children.begin(), children.end()); + EXPECT_EQUAL(StringVector({"extra", "regular"}), children); +} + +TEST_F("require that attributes are explorable", Fixture) +{ + EXPECT_TRUE(f._explorer.get_child("regular").get() != nullptr); + EXPECT_TRUE(f._explorer.get_child("extra").get() != nullptr); + EXPECT_TRUE(f._explorer.get_child("not").get() == nullptr); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore b/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore new file mode 100644 index 00000000000..45cd0a54f56 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_populator/.gitignore @@ -0,0 +1 @@ +searchcore_document_field_populator_test_app diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt new file mode 100644 index 00000000000..4c6da0a3397 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_populator/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_document_field_populator_test_app + SOURCES + document_field_populator_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_document_field_populator_test_app COMMAND searchcore_document_field_populator_test_app) diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/DESC b/searchcore/src/tests/proton/attribute/document_field_populator/DESC new file mode 100644 index 00000000000..cdc71250210 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_populator/DESC @@ -0,0 +1 @@ +document_field_populator test. Take a look at document_field_populator_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/FILES b/searchcore/src/tests/proton/attribute/document_field_populator/FILES new file mode 100644 index 00000000000..21f62452acf --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_populator/FILES @@ -0,0 +1 @@ +document_field_populator_test.cpp diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp new file mode 100644 index 00000000000..d0be50bfd2f --- /dev/null +++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("document_field_populator_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchcore/proton/attribute/document_field_populator.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/index/docbuilder.h> +#include <vespa/vespalib/util/stringfmt.h> + +using namespace document; +using namespace proton; +using namespace search; +using namespace search::index; + +typedef search::attribute::Config AVConfig; +typedef search::attribute::BasicType AVBasicType; + +Schema::AttributeField +createAttributeField() +{ + return Schema::AttributeField("a1", Schema::DataType::INT32); +} + +Schema +createSchema() +{ + Schema schema; + schema.addAttributeField(createAttributeField()); + return schema; +} + +struct DocContext +{ + Schema _schema; + DocBuilder _builder; + DocContext() + : _schema(createSchema()), + _builder(_schema) + { + } + Document::UP create(uint32_t id) { + vespalib::string docId = + vespalib::make_string("id:searchdocument:searchdocument::%u", id); + return _builder.startDocument(docId).endDocument(); + } +}; + +struct Fixture +{ + AttributeVector::SP _attr; + IntegerAttribute &_intAttr; + DocumentFieldPopulator _pop; + DocContext _ctx; + Fixture() + : _attr(search::AttributeFactory::createAttribute("a1", AVConfig(AVBasicType::INT32))), + _intAttr(dynamic_cast<IntegerAttribute &>(*_attr)), + _pop(createAttributeField(), _attr, "test"), + _ctx() + { + _intAttr.addDocs(2); + _intAttr.update(1, 100); + _intAttr.commit(); + } +}; + +TEST_F("require that document field is populated based on attribute content", Fixture) +{ + // NOTE: DocumentFieldRetriever (used by DocumentFieldPopulator) is fully tested + // with all data types in searchcore/src/tests/proton/server/documentretriever_test.cpp. + { + Document::UP doc = f._ctx.create(1); + f._pop.handleExisting(1, *doc); + EXPECT_EQUAL(100, doc->getValue("a1")->getAsInt()); + } +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore new file mode 100644 index 00000000000..f3666eecb6e --- /dev/null +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/.gitignore @@ -0,0 +1 @@ +searchcore_exclusive_attribute_read_accessor_test_app diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt new file mode 100644 index 00000000000..c39025ae39f --- /dev/null +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_exclusive_attribute_read_accessor_test_app + SOURCES + exclusive_attribute_read_accessor_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon +) +vespa_add_test(NAME searchcore_exclusive_attribute_read_accessor_test_app COMMAND searchcore_exclusive_attribute_read_accessor_test_app) diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC new file mode 100644 index 00000000000..ec5a407ddbd --- /dev/null +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/DESC @@ -0,0 +1 @@ +exclusive_attribute_read_accessor test. Take a look at exclusive_attribute_read_accessor_test.cpp for details. diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES new file mode 100644 index 00000000000..74a9ab77547 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/FILES @@ -0,0 +1 @@ +exclusive_attribute_read_accessor_test.cpp diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp new file mode 100644 index 00000000000..7cb6a503ae8 --- /dev/null +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> +#include <vespa/vespalib/util/sync.h> + +using namespace proton; +using namespace search; +using namespace vespalib; + +using ReadGuard = ExclusiveAttributeReadAccessor::Guard; + +AttributeVector::SP +createAttribute() +{ + attribute::Config cfg(attribute::BasicType::INT32, attribute::CollectionType::SINGLE); + return search::AttributeFactory::createAttribute("myattr", cfg); +} + +struct Fixture +{ + AttributeVector::SP attribute; + SequencedTaskExecutor writer; + ExclusiveAttributeReadAccessor accessor; + + Fixture() + : attribute(createAttribute()), + writer(1), + accessor(attribute, writer) + {} +}; + +TEST_F("require that attribute write thread is blocked while guard is held", Fixture) +{ + ReadGuard::UP guard = f.accessor.takeGuard(); + Gate gate; + f.writer.execute("myattr", [&gate]() { gate.countDown(); }); + bool reachedZero = gate.await(100); + EXPECT_FALSE(reachedZero); + EXPECT_EQUAL(1u, gate.getCount()); + + guard.reset(); + gate.await(); + EXPECT_EQUAL(0u, gate.getCount()); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore b/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/searchcore/src/tests/proton/attribute/gidmapattribute/.gitignore |