diff options
Diffstat (limited to 'searchcore/src/tests/proton/attribute/attribute_test.cpp')
-rw-r--r-- | searchcore/src/tests/proton/attribute/attribute_test.cpp | 835 |
1 files changed, 495 insertions, 340 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index 839ef14fcb0..91f580cd221 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -7,41 +7,43 @@ #include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/default_tensor_engine.h> +#include <vespa/eval/tensor/tensor.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> -#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchcore/proton/test/mock_attribute_manager.h> #include <vespa/searchcorespi/flush/iflushtarget.h> -#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> +#include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/bitvector_search_cache.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/searchlib/attribute/singlenumericattribute.hpp> #include <vespa/searchlib/common/idestructorcallback.h> -#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/predicate/predicate_hash.h> #include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/vespalib/btree/btreeroot.hpp> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcommon/attribute/iattributevector.h> -#include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/searchlib/attribute/singlenumericattribute.hpp> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/foregroundtaskexecutor.h> +#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h> #include <vespa/log/log.h> LOG_SETUP("attribute_test"); @@ -57,8 +59,11 @@ using namespace vespa::config::search; using proton::ImportedAttributesRepo; using proton::test::AttributeUtils; +using proton::test::MockAttributeManager; using search::TuneFileAttributes; using search::attribute::BitVectorSearchCache; +using search::attribute::DistanceMetric; +using search::attribute::HnswIndexParams; using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; using search::attribute::ImportedAttributeVectorFactory; @@ -67,24 +72,25 @@ using search::index::DummyFileHeaderContext; using search::index::schema::CollectionType; using search::predicate::PredicateHash; using search::predicate::PredicateIndex; +using search::tensor::DenseTensorAttribute; +using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::test::DirectoryHandler; using std::string; -using vespalib::eval::ValueType; -using vespalib::eval::TensorSpec; -using vespalib::tensor::Tensor; -using vespalib::tensor::DefaultTensorEngine; using vespalib::ForegroundTaskExecutor; using vespalib::SequencedTaskExecutorObserver; +using vespalib::eval::TensorSpec; +using vespalib::eval::ValueType; +using vespalib::tensor::DefaultTensorEngine; +using vespalib::tensor::Tensor; -using AVConfig = search::attribute::Config; using AVBasicType = search::attribute::BasicType; using AVCollectionType = search::attribute::CollectionType; +using AVConfig = search::attribute::Config; using Int32AttributeVector = SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> >; using LidVector = LidVectorContext::LidVector; -namespace -{ +namespace { const uint64_t createSerialNum = 42u; @@ -116,45 +122,48 @@ fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64 const std::shared_ptr<IDestructorCallback> emptyCallback; -struct Fixture -{ +class AttributeWriterTest : public ::testing::Test { +public: DirectoryHandler _dirHandler; - DummyFileHeaderContext _fileHeaderContext; - ForegroundTaskExecutor _attributeFieldWriterReal; - SequencedTaskExecutorObserver _attributeFieldWriter; - HwInfo _hwInfo; - proton::AttributeManager::SP _m; + std::unique_ptr<ForegroundTaskExecutor> _attributeFieldWriterReal; + std::unique_ptr<SequencedTaskExecutorObserver> _attributeFieldWriter; + std::shared_ptr<MockAttributeManager> _mgr; std::unique_ptr<AttributeWriter> _aw; - Fixture(uint32_t threads) + AttributeWriterTest() : _dirHandler(test_dir), - _fileHeaderContext(), - _attributeFieldWriterReal(threads), - _attributeFieldWriter(_attributeFieldWriterReal), - _hwInfo(), - _m(std::make_shared<proton::AttributeManager> - (test_dir, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, _attributeFieldWriter, _hwInfo)), + _attributeFieldWriterReal(), + _attributeFieldWriter(), + _mgr(), _aw() { - allocAttributeWriter(); + setup(1); } - Fixture() - : Fixture(1) - { + ~AttributeWriterTest(); + void setup(uint32_t threads) { + _aw.reset(); + _attributeFieldWriterReal = std::make_unique<ForegroundTaskExecutor>(threads); + _attributeFieldWriter = std::make_unique<SequencedTaskExecutorObserver>(*_attributeFieldWriterReal); + _mgr = std::make_shared<MockAttributeManager>(); + _mgr->set_writer(*_attributeFieldWriter); + allocAttributeWriter(); } - ~Fixture(); void allocAttributeWriter() { - _aw = std::make_unique<AttributeWriter>(_m); + _aw = std::make_unique<AttributeWriter>(_mgr); } AttributeVector::SP addAttribute(const vespalib::string &name) { - return addAttribute({name, AVConfig(AVBasicType::INT32)}, createSerialNum); + return addAttribute({name, AVConfig(AVBasicType::INT32)}); } - AttributeVector::SP addAttribute(const AttributeSpec &spec, SerialNum serialNum) { - auto ret = _m->addAttribute(spec, serialNum); + AttributeVector::SP addAttribute(const AttributeSpec &spec) { + auto ret = _mgr->addAttribute(spec.getName(), + AttributeFactory::createAttribute(spec.getName(), spec.getConfig())); allocAttributeWriter(); return ret; } + void add_attribute(AttributeVector::SP attr) { + _mgr->addAttribute(attr->getName(), std::move(attr)); + allocAttributeWriter(); + } void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit = true) { _aw->put(serialNum, doc, lid, immediateCommit, emptyCallback); @@ -177,13 +186,13 @@ struct Fixture _aw->forceCommit(serialNum, emptyCallback); } void assertExecuteHistory(std::vector<uint32_t> expExecuteHistory) { - EXPECT_EQUAL(expExecuteHistory, _attributeFieldWriter.getExecuteHistory()); + EXPECT_EQ(expExecuteHistory, _attributeFieldWriter->getExecuteHistory()); } }; -Fixture::~Fixture() = default; +AttributeWriterTest::~AttributeWriterTest() = default; -TEST_F("require that attribute writer handles put", Fixture) +TEST_F(AttributeWriterTest, handles_put) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); @@ -193,108 +202,108 @@ TEST_F("require that attribute writer handles put", Fixture) DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); - AttributeVector::SP a3 = f.addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}, createSerialNum); - AttributeVector::SP a4 = f.addAttribute({"a4", AVConfig(AVBasicType::STRING)}, createSerialNum); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute({"a2", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); + auto a3 = addAttribute({"a3", AVConfig(AVBasicType::FLOAT)}); + auto a4 = addAttribute({"a4", AVConfig(AVBasicType::STRING)}); attribute::IntegerContent ibuf; attribute::FloatContent fbuf; attribute::ConstCharContent sbuf; { // empty document should give default values - EXPECT_EQUAL(1u, a1->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(2u, a2->getNumDocs()); - EXPECT_EQUAL(2u, a3->getNumDocs()); - EXPECT_EQUAL(2u, a4->getNumDocs()); - EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, a4->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a1->getNumDocs()); + put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(2u, a2->getNumDocs()); + EXPECT_EQ(2u, a3->getNumDocs()); + EXPECT_EQ(2u, a4->getNumDocs()); + EXPECT_EQ(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 1); - EXPECT_EQUAL(1u, ibuf.size()); + EXPECT_EQ(1u, ibuf.size()); EXPECT_TRUE(search::attribute::isUndefined<int32_t>(ibuf[0])); ibuf.fill(*a2, 1); - EXPECT_EQUAL(0u, ibuf.size()); + EXPECT_EQ(0u, ibuf.size()); fbuf.fill(*a3, 1); - EXPECT_EQUAL(1u, fbuf.size()); + EXPECT_EQ(1u, fbuf.size()); EXPECT_TRUE(search::attribute::isUndefined<float>(fbuf[0])); sbuf.fill(*a4, 1); - EXPECT_EQUAL(1u, sbuf.size()); - EXPECT_EQUAL(strcmp("", sbuf[0]), 0); + EXPECT_EQ(1u, sbuf.size()); + EXPECT_EQ(strcmp("", sbuf[0]), 0); } { // document with single value & multi value attribute - Document::UP doc = idb.startDocument("id:ns:searchdocument::2"). + auto doc = idb.startDocument("id:ns:searchdocument::2"). startAttributeField("a1").addInt(10).endField(). startAttributeField("a2").startElement().addInt(20).endElement(). startElement().addInt(30).endElement().endField().endDocument(); - f.put(2, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a2->getNumDocs()); - EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(2u, a4->getStatus().getLastSyncToken()); + put(2, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a2->getNumDocs()); + EXPECT_EQ(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(2u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 2); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(10u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(10u, ibuf[0]); ibuf.fill(*a2, 2); - EXPECT_EQUAL(2u, ibuf.size()); - EXPECT_EQUAL(20u, ibuf[0]); - EXPECT_EQUAL(30u, ibuf[1]); + EXPECT_EQ(2u, ibuf.size()); + EXPECT_EQ(20u, ibuf[0]); + EXPECT_EQ(30u, ibuf[1]); } { // replace existing document - Document::UP doc = idb.startDocument("id:ns:searchdocument::2"). + auto doc = idb.startDocument("id:ns:searchdocument::2"). startAttributeField("a1").addInt(100).endField(). startAttributeField("a2").startElement().addInt(200).endElement(). startElement().addInt(300).endElement(). startElement().addInt(400).endElement().endField().endDocument(); - f.put(3, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a2->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a2->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a3->getStatus().getLastSyncToken()); - EXPECT_EQUAL(3u, a4->getStatus().getLastSyncToken()); + put(3, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a2->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a2->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a3->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a4->getStatus().getLastSyncToken()); ibuf.fill(*a1, 2); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(100u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(100u, ibuf[0]); ibuf.fill(*a2, 2); - EXPECT_EQUAL(3u, ibuf.size()); - EXPECT_EQUAL(200u, ibuf[0]); - EXPECT_EQUAL(300u, ibuf[1]); - EXPECT_EQUAL(400u, ibuf[2]); + EXPECT_EQ(3u, ibuf.size()); + EXPECT_EQ(200u, ibuf[0]); + EXPECT_EQ(300u, ibuf[1]); + EXPECT_EQ(400u, ibuf[2]); } } -TEST_F("require that attribute writer handles predicate put", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_put) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); // empty document should give default values - EXPECT_EQUAL(1u, a1->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(1u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, a1->getNumDocs()); + put(1, *idb.startDocument("id:ns:searchdocument::1").endDocument(), 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(1u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); // document with single value attribute PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::2").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(2, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(2u, a1->getStatus().getLastSyncToken()); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + put(2, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(2u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); auto it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); EXPECT_FALSE(it.valid()); @@ -303,9 +312,9 @@ TEST_F("require that attribute writer handles predicate put", Fixture) doc = idb.startDocument("id:ns:searchdocument::2").startAttributeField("a1") .addPredicate(builder.feature("foo").value("bar").build()) .endField().endDocument(); - f.put(3, *doc, 2); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + put(3, *doc, 2); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); it = index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")); EXPECT_TRUE(it.valid()); @@ -317,21 +326,21 @@ assertUndefined(const IAttributeVector &attr, uint32_t docId) EXPECT_TRUE(search::attribute::isUndefined<int32_t>(attr.getInt(docId))); } -TEST_F("require that attribute writer handles remove", Fixture) +TEST_F(AttributeWriterTest, handles_remove) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); - f.remove(2, 0); + remove(2, 0); - TEST_DO(assertUndefined(*a1, 0)); - TEST_DO(assertUndefined(*a2, 0)); + assertUndefined(*a1, 0); + assertUndefined(*a2, 0); - f.remove(2, 0); // same sync token as previous + remove(2, 0); // same sync token as previous try { - f.remove(1, 0); // lower sync token than previous + remove(1, 0); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -339,62 +348,63 @@ TEST_F("require that attribute writer handles remove", Fixture) } } -TEST_F("require that attribute writer handles batch remove", Fixture) +TEST_F(AttributeWriterTest, handles_batch_remove) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 4, 22, 1); fillAttribute(a2, 4, 33, 1); LidVector lidsToRemove = {1,3}; - f.remove(lidsToRemove, 2); - - TEST_DO(assertUndefined(*a1, 1)); - EXPECT_EQUAL(22, a1->getInt(2)); - TEST_DO(assertUndefined(*a1, 3)); - TEST_DO(assertUndefined(*a2, 1)); - EXPECT_EQUAL(33, a2->getInt(2)); - TEST_DO(assertUndefined(*a2, 3)); + remove(lidsToRemove, 2); + + assertUndefined(*a1, 1); + EXPECT_EQ(22, a1->getInt(2)); + assertUndefined(*a1, 3); + assertUndefined(*a2, 1); + EXPECT_EQ(33, a2->getInt(2)); + assertUndefined(*a2, 3); } -void verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) +void +verifyAttributeContent(const AttributeVector & v, uint32_t lid, vespalib::stringref expected) { attribute::ConstCharContent sbuf; sbuf.fill(v, lid); - EXPECT_EQUAL(1u, sbuf.size()); - EXPECT_EQUAL(expected, sbuf[0]); + EXPECT_EQ(1u, sbuf.size()); + EXPECT_EQ(expected, sbuf[0]); } -TEST_F("require that visibilitydelay is honoured", Fixture) +TEST_F(AttributeWriterTest, visibility_delay_is_honoured) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::STRING)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::STRING)}); Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::STRING, CollectionType::SINGLE)); DocBuilder idb(s); - EXPECT_EQUAL(1u, a1->getNumDocs()); - EXPECT_EQUAL(0u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(1u, a1->getNumDocs()); + EXPECT_EQ(0u, a1->getStatus().getLastSyncToken()); Document::UP doc = idb.startDocument("id:ns:searchdocument::1") .startAttributeField("a1").addStr("10").endField() .endDocument(); - f.put(3, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); - AttributeWriter awDelayed(f._m); + put(3, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); + AttributeWriter awDelayed(_mgr); awDelayed.put(4, *doc, 2, false, emptyCallback); - EXPECT_EQUAL(3u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(3u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); awDelayed.put(5, *doc, 4, false, emptyCallback); - EXPECT_EQUAL(5u, a1->getNumDocs()); - EXPECT_EQUAL(3u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(5u, a1->getNumDocs()); + EXPECT_EQ(3u, a1->getStatus().getLastSyncToken()); awDelayed.forceCommit(6, emptyCallback); - EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(6u, a1->getStatus().getLastSyncToken()); - AttributeWriter awDelayedShort(f._m); + AttributeWriter awDelayedShort(_mgr); awDelayedShort.put(7, *doc, 2, false, emptyCallback); - EXPECT_EQUAL(6u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(6u, a1->getStatus().getLastSyncToken()); awDelayedShort.put(8, *doc, 2, false, emptyCallback); awDelayedShort.forceCommit(8, emptyCallback); - EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); awDelayed.put(9, *idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1").addStr("11").endField().endDocument(), @@ -403,40 +413,39 @@ TEST_F("require that visibilitydelay is honoured", Fixture) 2, false, emptyCallback); awDelayed.put(11, *idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1").addStr("30").endField().endDocument(), 2, false, emptyCallback); - EXPECT_EQUAL(8u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(8u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "10"); awDelayed.forceCommit(12, emptyCallback); - EXPECT_EQUAL(12u, a1->getStatus().getLastSyncToken()); + EXPECT_EQ(12u, a1->getStatus().getLastSyncToken()); verifyAttributeContent(*a1, 2, "30"); - } -TEST_F("require that attribute writer handles predicate remove", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_remove) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); Schema s; s.addAttributeField( Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(s); PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); - f.remove(2, 1); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); + remove(2, 1); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); } -TEST_F("require that attribute writer handles update", Fixture) +TEST_F(AttributeWriterTest, handles_update) { - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); + auto a1 = addAttribute("a1"); + auto a2 = addAttribute("a2"); fillAttribute(a1, 1, 10, 1); fillAttribute(a2, 1, 20, 1); @@ -454,19 +463,19 @@ TEST_F("require that attribute writer handles update", Fixture) DummyFieldUpdateCallback onUpdate; bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit, onUpdate); + update(2, upd, 1, immediateCommit, onUpdate); attribute::IntegerContent ibuf; ibuf.fill(*a1, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(15u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(15u, ibuf[0]); ibuf.fill(*a2, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(30u, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(30u, ibuf[0]); - f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous + update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous try { - f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous + update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -474,20 +483,20 @@ TEST_F("require that attribute writer handles update", Fixture) } } -TEST_F("require that attribute writer handles predicate update", Fixture) +TEST_F(AttributeWriterTest, handles_predicate_update) { - AttributeVector::SP a1 = f.addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}, createSerialNum); + auto a1 = addAttribute({"a1", AVConfig(AVBasicType::PREDICATE)}); Schema schema; schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::BOOLEANTREE, CollectionType::SINGLE)); DocBuilder idb(schema); PredicateSlimeBuilder builder; - Document::UP doc = + auto doc = idb.startDocument("id:ns:searchdocument::1").startAttributeField("a1") .addPredicate(builder.true_predicate().build()) .endField().endDocument(); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); const document::DocumentType &dt(idb.getDocumentType()); DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("id:ns:searchdocument::1")); @@ -496,20 +505,20 @@ TEST_F("require that attribute writer handles predicate update", Fixture) .addUpdate(AssignValueUpdate(new_value))); PredicateIndex &index = static_cast<PredicateAttribute &>(*a1).getIndex(); - EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); + EXPECT_EQ(1u, index.getZeroConstraintDocs().size()); EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); bool immediateCommit = true; DummyFieldUpdateCallback onUpdate; - f.update(2, upd, 1, immediateCommit, onUpdate); - EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); + update(2, upd, 1, immediateCommit, onUpdate); + EXPECT_EQ(0u, index.getZeroConstraintDocs().size()); EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); } -struct AttributeCollectionSpecFixture -{ +class AttributeCollectionSpecTest : public ::testing::Test { +public: AttributesConfigBuilder _builder; AttributeCollectionSpecFactory _factory; - AttributeCollectionSpecFixture(bool fastAccessOnly) + AttributeCollectionSpecTest(bool fastAccessOnly) : _builder(), _factory(search::GrowStrategy(), 100, fastAccessOnly) { @@ -528,49 +537,47 @@ struct AttributeCollectionSpecFixture } }; -struct NormalAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture -{ - NormalAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(false) {} +class NormalAttributeCollectionSpecTest : public AttributeCollectionSpecTest { +public: + NormalAttributeCollectionSpecTest() : AttributeCollectionSpecTest(false) {} }; -struct FastAccessAttributeCollectionSpecFixture : public AttributeCollectionSpecFixture +struct FastAccessAttributeCollectionSpecTest : public AttributeCollectionSpecTest { - FastAccessAttributeCollectionSpecFixture() : AttributeCollectionSpecFixture(true) {} + FastAccessAttributeCollectionSpecTest() : AttributeCollectionSpecTest(true) {} }; -TEST_F("require that normal attribute collection spec can be created", - NormalAttributeCollectionSpecFixture) +TEST_F(NormalAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = f.create(10, 20); - EXPECT_EQUAL(2u, spec->getAttributes().size()); - EXPECT_EQUAL("a1", spec->getAttributes()[0].getName()); - EXPECT_EQUAL("a2", spec->getAttributes()[1].getName()); - EXPECT_EQUAL(10u, spec->getDocIdLimit()); - EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); + AttributeCollectionSpec::UP spec = create(10, 20); + EXPECT_EQ(2u, spec->getAttributes().size()); + EXPECT_EQ("a1", spec->getAttributes()[0].getName()); + EXPECT_EQ("a2", spec->getAttributes()[1].getName()); + EXPECT_EQ(10u, spec->getDocIdLimit()); + EXPECT_EQ(20u, spec->getCurrentSerialNum()); } -TEST_F("require that fast access attribute collection spec can be created", - FastAccessAttributeCollectionSpecFixture) +TEST_F(FastAccessAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = f.create(10, 20); - EXPECT_EQUAL(1u, spec->getAttributes().size()); - EXPECT_EQUAL("a2", spec->getAttributes()[0].getName()); - EXPECT_EQUAL(10u, spec->getDocIdLimit()); - EXPECT_EQUAL(20u, spec->getCurrentSerialNum()); + AttributeCollectionSpec::UP spec = create(10, 20); + EXPECT_EQ(1u, spec->getAttributes().size()); + EXPECT_EQ("a2", spec->getAttributes()[0].getName()); + EXPECT_EQ(10u, spec->getDocIdLimit()); + EXPECT_EQ(20u, spec->getCurrentSerialNum()); } const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"}; -struct FilterFixture -{ +class FilterAttributeManagerTest : public ::testing::Test { +public: DirectoryHandler _dirHandler; DummyFileHeaderContext _fileHeaderContext; ForegroundTaskExecutor _attributeFieldWriter; HwInfo _hwInfo; - proton::AttributeManager::SP _baseMgr; FilterAttributeManager _filterMgr; - FilterFixture() + + FilterAttributeManagerTest() : _dirHandler(test_dir), _fileHeaderContext(), _attributeFieldWriter(), @@ -587,34 +594,34 @@ struct FilterFixture } }; -TEST_F("require that filter attribute manager can filter attributes", FilterFixture) +TEST_F(FilterAttributeManagerTest, filter_attributes) { - EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == nullptr); - EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != nullptr); + EXPECT_TRUE(_filterMgr.getAttribute("a1").get() == nullptr); + EXPECT_TRUE(_filterMgr.getAttribute("a2").get() != nullptr); std::vector<AttributeGuard> attrs; - f._filterMgr.getAttributeList(attrs); - EXPECT_EQUAL(1u, attrs.size()); - EXPECT_EQUAL("a2", attrs[0]->getName()); - searchcorespi::IFlushTarget::List targets = f._filterMgr.getFlushTargets(); - EXPECT_EQUAL(2u, targets.size()); - EXPECT_EQUAL("attribute.flush.a2", targets[0]->getName()); - EXPECT_EQUAL("attribute.shrink.a2", targets[1]->getName()); + _filterMgr.getAttributeList(attrs); + EXPECT_EQ(1u, attrs.size()); + EXPECT_EQ("a2", attrs[0]->getName()); + searchcorespi::IFlushTarget::List targets = _filterMgr.getFlushTargets(); + EXPECT_EQ(2u, targets.size()); + EXPECT_EQ("attribute.flush.a2", targets[0]->getName()); + EXPECT_EQ("attribute.shrink.a2", targets[1]->getName()); } -TEST_F("require that filter attribute manager can return flushed serial number", FilterFixture) +TEST_F(FilterAttributeManagerTest, returns_flushed_serial_number) { - f._baseMgr->flushAll(100); - EXPECT_EQUAL(0u, f._filterMgr.getFlushedSerialNum("a1")); - EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2")); + _baseMgr->flushAll(100); + EXPECT_EQ(0u, _filterMgr.getFlushedSerialNum("a1")); + EXPECT_EQ(100u, _filterMgr.getFlushedSerialNum("a2")); } -TEST_F("readable_attribute_vector filters attributes", FilterFixture) +TEST_F(FilterAttributeManagerTest, readable_attribute_vector_filters_attributes) { - auto av = f._filterMgr.readable_attribute_vector("a2"); + auto av = _filterMgr.readable_attribute_vector("a2"); ASSERT_TRUE(av); - EXPECT_EQUAL("a2", av->makeReadGuard(false)->attribute()->getName()); + EXPECT_EQ("a2", av->makeReadGuard(false)->attribute()->getName()); - av = f._filterMgr.readable_attribute_vector("a1"); + av = _filterMgr.readable_attribute_vector("a1"); EXPECT_FALSE(av); } @@ -625,18 +632,20 @@ Tensor::UP make_tensor(const TensorSpec &spec) { return Tensor::UP(dynamic_cast<Tensor*>(tensor.release())); } +const vespalib::string sparse_tensor = "tensor(x{},y{})"; + AttributeVector::SP -createTensorAttribute(Fixture &f) { +createTensorAttribute(AttributeWriterTest &t) { AVConfig cfg(AVBasicType::TENSOR); - cfg.setTensorType(ValueType::from_spec("tensor(x{},y{})")); - auto ret = f.addAttribute({"a1", cfg}, createSerialNum); + cfg.setTensorType(ValueType::from_spec(sparse_tensor)); + auto ret = t.addAttribute({"a1", cfg}); return ret; } Schema -createTensorSchema() { +createTensorSchema(const vespalib::string& tensor_spec = sparse_tensor) { Schema schema; - schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::TENSOR, CollectionType::SINGLE)); + schema.addAttributeField(Schema::AttributeField("a1", schema::DataType::TENSOR, CollectionType::SINGLE, tensor_spec)); return schema; } @@ -649,38 +658,34 @@ createTensorPutDoc(DocBuilder &builder, const Tensor &tensor) { } - -TEST_F("Test that we can use attribute writer to write to tensor attribute", - Fixture) +TEST_F(AttributeWriterTest, can_write_to_tensor_attribute) { - AttributeVector::SP a1 = createTensorAttribute(f); + auto a1 = createTensorAttribute(*this); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "4"}, {"y", "5"}}, 7)); Document::UP doc = createTensorPutDoc(builder, *tensor); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - TensorAttribute *tensorAttribute = - dynamic_cast<TensorAttribute *>(a1.get()); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + auto *tensorAttribute = dynamic_cast<TensorAttribute *>(a1.get()); EXPECT_TRUE(tensorAttribute != nullptr); auto tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); EXPECT_TRUE(tensor->equals(*tensor2)); } -TEST_F("require that attribute writer handles tensor assign update", Fixture) +TEST_F(AttributeWriterTest, handles_tensor_assign_update) { - AttributeVector::SP a1 = createTensorAttribute(f); + auto a1 = createTensorAttribute(*this); Schema s = createTensorSchema(); DocBuilder builder(s); - auto tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "6"}, {"y", "7"}}, 9)); - Document::UP doc = createTensorPutDoc(builder, *tensor); - f.put(1, *doc, 1); - EXPECT_EQUAL(2u, a1->getNumDocs()); - TensorAttribute *tensorAttribute = - dynamic_cast<TensorAttribute *>(a1.get()); + auto doc = createTensorPutDoc(builder, *tensor); + put(1, *doc, 1); + EXPECT_EQ(2u, a1->getNumDocs()); + auto *tensorAttribute = dynamic_cast<TensorAttribute *>(a1.get()); EXPECT_TRUE(tensorAttribute != nullptr); auto tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); @@ -688,23 +693,22 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) const document::DocumentType &dt(builder.getDocumentType()); DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("id:ns:searchdocument::1")); - auto new_tensor = make_tensor(TensorSpec("tensor(x{},y{})") + auto new_tensor = make_tensor(TensorSpec(sparse_tensor) .add({{"x", "8"}, {"y", "9"}}, 11)); - TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})")); + TensorDataType xySparseTensorDataType(vespalib::eval::ValueType::from_spec(sparse_tensor)); TensorFieldValue new_value(xySparseTensorDataType); new_value = new_tensor->clone(); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); bool immediateCommit = true; DummyFieldUpdateCallback onUpdate; - f.update(2, upd, 1, immediateCommit, onUpdate); - EXPECT_EQUAL(2u, a1->getNumDocs()); + update(2, upd, 1, immediateCommit, onUpdate); + EXPECT_EQ(2u, a1->getNumDocs()); EXPECT_TRUE(tensorAttribute != nullptr); tensor2 = tensorAttribute->getTensor(1); EXPECT_TRUE(static_cast<bool>(tensor2)); EXPECT_TRUE(!tensor->equals(*tensor2)); EXPECT_TRUE(new_tensor->equals(*tensor2)); - } namespace { @@ -712,16 +716,16 @@ namespace { void assertPutDone(AttributeVector &attr, int32_t expVal) { - EXPECT_EQUAL(2u, attr.getNumDocs()); - EXPECT_EQUAL(1u, attr.getStatus().getLastSyncToken()); + EXPECT_EQ(2u, attr.getNumDocs()); + EXPECT_EQ(1u, attr.getStatus().getLastSyncToken()); attribute::IntegerContent ibuf; ibuf.fill(attr, 1); - EXPECT_EQUAL(1u, ibuf.size()); - EXPECT_EQUAL(expVal, ibuf[0]); + EXPECT_EQ(1u, ibuf.size()); + EXPECT_EQ(expVal, ibuf[0]); } void -putAttributes(Fixture &f, std::vector<uint32_t> expExecuteHistory) +putAttributes(AttributeWriterTest &t, std::vector<uint32_t> expExecuteHistory) { Schema s; s.addAttributeField(Schema::AttributeField("a1", schema::DataType::INT32, CollectionType::SINGLE)); @@ -730,41 +734,205 @@ putAttributes(Fixture &f, std::vector<uint32_t> expExecuteHistory) DocBuilder idb(s); - AttributeVector::SP a1 = f.addAttribute("a1"); - AttributeVector::SP a2 = f.addAttribute("a2"); - AttributeVector::SP a3 = f.addAttribute("a3"); + auto a1 = t.addAttribute("a1"); + auto a2 = t.addAttribute("a2"); + auto a3 = t.addAttribute("a3"); - EXPECT_EQUAL(1u, a1->getNumDocs()); - EXPECT_EQUAL(1u, a2->getNumDocs()); - EXPECT_EQUAL(1u, a3->getNumDocs()); - f.put(1, *idb.startDocument("id:ns:searchdocument::1"). + EXPECT_EQ(1u, a1->getNumDocs()); + EXPECT_EQ(1u, a2->getNumDocs()); + EXPECT_EQ(1u, a3->getNumDocs()); + t.put(1, *idb.startDocument("id:ns:searchdocument::1"). startAttributeField("a1").addInt(10).endField(). startAttributeField("a2").addInt(15).endField(). startAttributeField("a3").addInt(20).endField(). endDocument(), 1); - TEST_DO(assertPutDone(*a1, 10)); - TEST_DO(assertPutDone(*a2, 15)); - TEST_DO(assertPutDone(*a3, 20)); - TEST_DO(f.assertExecuteHistory(expExecuteHistory)); + assertPutDone(*a1, 10); + assertPutDone(*a2, 15); + assertPutDone(*a3, 20); + t.assertExecuteHistory(expExecuteHistory); +} + +} + +TEST_F(AttributeWriterTest, spreads_write_over_1_write_context) +{ + putAttributes(*this, {0}); } +TEST_F(AttributeWriterTest, spreads_write_over_2_write_contexts) +{ + setup(2); + putAttributes(*this, {0, 1}); } -TEST_F("require that attribute writer spreads write over 1 write context", Fixture(1)) +TEST_F(AttributeWriterTest, spreads_write_over_3_write_contexts) { - TEST_DO(putAttributes(f, {0})); + setup(8); + putAttributes(*this, {0, 1, 2}); } -TEST_F("require that attribute writer spreads write over 2 write contexts", Fixture(2)) +struct MockPrepareResult : public PrepareResult { + uint32_t docid; + const Tensor& tensor; + MockPrepareResult(uint32_t docid_in, const Tensor& tensor_in) : docid(docid_in), tensor(tensor_in) {} +}; + +class MockDenseTensorAttribute : public DenseTensorAttribute { +public: + mutable size_t prepare_set_tensor_cnt; + mutable size_t complete_set_tensor_cnt; + size_t clear_doc_cnt; + + MockDenseTensorAttribute(vespalib::stringref name, const AVConfig& cfg) + : DenseTensorAttribute(name, cfg), + prepare_set_tensor_cnt(0), + complete_set_tensor_cnt(0), + clear_doc_cnt(0) + {} + uint32_t clearDoc(DocId docid) override { + ++clear_doc_cnt; + return DenseTensorAttribute::clearDoc(docid); + } + std::unique_ptr<PrepareResult> prepare_set_tensor(uint32_t docid, const Tensor& tensor) const override { + ++prepare_set_tensor_cnt; + return std::make_unique<MockPrepareResult>(docid, tensor); + } + + virtual void complete_set_tensor(DocId docid, const Tensor& tensor, std::unique_ptr<PrepareResult> prepare_result) override { + ++complete_set_tensor_cnt; + assert(prepare_result); + auto* mock_result = dynamic_cast<MockPrepareResult*>(prepare_result.get()); + assert(mock_result); + EXPECT_EQ(docid, mock_result->docid); + EXPECT_EQ(tensor, mock_result->tensor); + } +}; + +const vespalib::string dense_tensor = "tensor(x[2])"; + +AVConfig +get_tensor_config(bool allow_multi_threaded_indexing) { - TEST_DO(putAttributes(f, {0, 1})); + AVConfig cfg(AVBasicType::TENSOR); + cfg.setTensorType(ValueType::from_spec(dense_tensor)); + cfg.set_hnsw_index_params(HnswIndexParams(4, 4, DistanceMetric::Euclidean, allow_multi_threaded_indexing)); + return cfg; } -TEST_F("require that attribute writer spreads write over 3 write contexts", Fixture(8)) +std::shared_ptr<MockDenseTensorAttribute> +make_mock_tensor_attribute(const vespalib::string& name, bool allow_multi_threaded_indexing) { - TEST_DO(putAttributes(f, {0, 1, 2})); + auto cfg = get_tensor_config(allow_multi_threaded_indexing); + return std::make_shared<MockDenseTensorAttribute>(name, cfg); } +TEST_F(AttributeWriterTest, tensor_attributes_using_two_phase_put_are_in_separate_write_contexts) +{ + addAttribute("a1"); + addAttribute({"t1", get_tensor_config(true)}); + addAttribute({"t2", get_tensor_config(true)}); + addAttribute({"t3", get_tensor_config(false)}); + allocAttributeWriter(); + + const auto& ctx = _aw->get_write_contexts(); + EXPECT_EQ(3, ctx.size()); + EXPECT_FALSE(ctx[0].use_two_phase_put()); + EXPECT_EQ(2, ctx[0].getFields().size()); + + EXPECT_TRUE(ctx[1].use_two_phase_put()); + EXPECT_EQ(1, ctx[1].getFields().size()); + EXPECT_EQ("t1", ctx[1].getFields()[0].getAttribute().getName()); + + EXPECT_TRUE(ctx[2].use_two_phase_put()); + EXPECT_EQ(1, ctx[2].getFields().size()); + EXPECT_EQ("t2", ctx[2].getFields()[0].getAttribute().getName()); +} + +class TwoPhasePutTest : public AttributeWriterTest { +public: + Schema schema; + DocBuilder builder; + std::shared_ptr<MockDenseTensorAttribute> attr; + std::unique_ptr<Tensor> tensor; + + TwoPhasePutTest() + : AttributeWriterTest(), + schema(createTensorSchema(dense_tensor)), + builder(schema), + attr() + { + setup(2); + attr = make_mock_tensor_attribute("a1", true); + add_attribute(attr); + AttributeManager::padAttribute(*attr, 4); + attr->clear_doc_cnt = 0; + tensor = make_tensor(TensorSpec(dense_tensor) + .add({{"x", 0}}, 3).add({{"x", 1}}, 5)); + } + void expect_tensor_attr_calls(size_t exp_prepare_cnt, + size_t exp_complete_cnt, + size_t exp_clear_doc_cnt = 0) { + EXPECT_EQ(exp_prepare_cnt, attr->prepare_set_tensor_cnt); + EXPECT_EQ(exp_complete_cnt, attr->complete_set_tensor_cnt); + EXPECT_EQ(exp_clear_doc_cnt, attr->clear_doc_cnt); + } + Document::UP make_doc() { + return createTensorPutDoc(builder, *tensor); + } + Document::UP make_no_field_doc() { + return builder.startDocument("id:ns:searchdocument::1").endDocument(); + } + Document::UP make_no_tensor_doc() { + return builder.startDocument("id:ns:searchdocument::1"). + startAttributeField("a1"). + addTensor(std::unique_ptr<vespalib::tensor::Tensor>()).endField().endDocument(); + } +}; + +TEST_F(TwoPhasePutTest, handles_put_in_two_phases_when_specified_for_tensor_attribute) +{ + auto doc = make_doc(); + + put(1, *doc, 1); + expect_tensor_attr_calls(1, 1); + assertExecuteHistory({1, 0}); + + put(2, *doc, 2); + expect_tensor_attr_calls(2, 2); + assertExecuteHistory({1, 0, 0, 0}); + + put(3, *doc, 3); + expect_tensor_attr_calls(3, 3); + // Note that the prepare step is executed round-robin between the 2 threads. + assertExecuteHistory({1, 0, 0, 0, 1, 0}); +} + +TEST_F(TwoPhasePutTest, put_is_ignored_when_serial_number_is_older_or_equal_to_attribute) +{ + auto doc = make_doc(); + attr->commit(7, 7); + put(7, *doc, 1); + expect_tensor_attr_calls(0, 0); + assertExecuteHistory({1, 0}); +} + +TEST_F(TwoPhasePutTest, document_is_cleared_if_field_is_not_set) +{ + auto doc = make_no_field_doc(); + put(1, *doc, 1); + expect_tensor_attr_calls(0, 0, 1); + assertExecuteHistory({1, 0}); +} + +TEST_F(TwoPhasePutTest, document_is_cleared_if_tensor_in_field_is_not_set) +{ + auto doc = make_no_tensor_doc(); + put(1, *doc, 1); + expect_tensor_attr_calls(0, 0, 1); + assertExecuteHistory({1, 0}); +} + + ImportedAttributeVector::SP createImportedAttribute(const vespalib::string &name) { @@ -787,74 +955,66 @@ createImportedAttributesRepo() return result; } -TEST_F("require that AttributeWriter::forceCommit() clears search cache in imported attribute vectors", Fixture) +TEST_F(AttributeWriterTest, forceCommit_clears_search_cache_in_imported_attribute_vectors) { - f._m->setImportedAttributes(createImportedAttributesRepo()); - f.commit(10); - EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); - EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); + _mgr->setImportedAttributes(createImportedAttributesRepo()); + commit(10); + EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_a")->getSearchCache()->size()); + EXPECT_EQ(0u, _mgr->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } -struct StructFixtureBase : public Fixture -{ +class StructWriterTestBase : public AttributeWriterTest { +public: DocumentType _type; const Field _valueField; StructDataType _structFieldType; - StructFixtureBase() - : Fixture(), + StructWriterTestBase() + : AttributeWriterTest(), _type("test"), _valueField("value", 2, *DataType::INT, true), _structFieldType("struct") { - addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum); + addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}); _type.addField(_valueField); _structFieldType.addField(_valueField); } - ~StructFixtureBase(); + ~StructWriterTestBase(); - std::unique_ptr<StructFieldValue> - makeStruct() - { + std::unique_ptr<StructFieldValue> makeStruct() { return std::make_unique<StructFieldValue>(_structFieldType); } - std::unique_ptr<StructFieldValue> - makeStruct(const int32_t value) - { + std::unique_ptr<StructFieldValue> makeStruct(const int32_t value) { auto ret = makeStruct(); ret->setValue(_valueField, IntFieldValue(value)); return ret; } - std::unique_ptr<Document> - makeDoc() - { + std::unique_ptr<Document> makeDoc() { return std::make_unique<Document>(_type, DocumentId("id::test::1")); } }; -StructFixtureBase::~StructFixtureBase() = default; +StructWriterTestBase::~StructWriterTestBase() = default; -struct StructArrayFixture : public StructFixtureBase -{ - using StructFixtureBase::makeDoc; +class StructArrayWriterTest : public StructWriterTestBase { +public: + using StructWriterTestBase::makeDoc; const ArrayDataType _structArrayFieldType; const Field _structArrayField; - StructArrayFixture() - : StructFixtureBase(), + StructArrayWriterTest() + : StructWriterTestBase(), _structArrayFieldType(_structFieldType), _structArrayField("array", _structArrayFieldType, true) { - addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); _type.addField(_structArrayField); } - ~StructArrayFixture(); + ~StructArrayWriterTest(); - std::unique_ptr<Document> - makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) - { + std::unique_ptr<Document> makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) { auto doc = makeDoc(); doc->setValue(_valueField, IntFieldValue(value)); ArrayFieldValue s(_structArrayFieldType); @@ -865,49 +1025,47 @@ struct StructArrayFixture : public StructFixtureBase return doc; } void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) { - auto valueAttr = _m->getAttribute("value")->getSP(); - auto arrayValueAttr = _m->getAttribute("array.value")->getSP(); - EXPECT_EQUAL(value, valueAttr->getInt(lid)); + auto valueAttr = _mgr->getAttribute("value")->getSP(); + auto arrayValueAttr = _mgr->getAttribute("array.value")->getSP(); + EXPECT_EQ(value, valueAttr->getInt(lid)); attribute::IntegerContent ibuf; ibuf.fill(*arrayValueAttr, lid); - EXPECT_EQUAL(arrayValues.size(), ibuf.size()); + EXPECT_EQ(arrayValues.size(), ibuf.size()); for (size_t i = 0; i < arrayValues.size(); ++i) { - EXPECT_EQUAL(arrayValues[i], ibuf[i]); + EXPECT_EQ(arrayValues[i], ibuf[i]); } } }; -StructArrayFixture::~StructArrayFixture() = default; +StructArrayWriterTest::~StructArrayWriterTest() = default; -TEST_F("require that update with doc argument updates struct field attributes (array)", StructArrayFixture) +TEST_F(StructArrayWriterTest, update_with_doc_argument_updates_struct_field_attributes) { - auto doc = f.makeDoc(10, {11, 12}); - f.put(10, *doc, 1); - TEST_DO(f.checkAttrs(1, 10, {11, 12})); - doc = f.makeDoc(20, {21}); - f.update(11, *doc, 1, true); - TEST_DO(f.checkAttrs(1, 10, {21})); + auto doc = makeDoc(10, {11, 12}); + put(10, *doc, 1); + checkAttrs(1, 10, {11, 12}); + doc = makeDoc(20, {21}); + update(11, *doc, 1, true); + checkAttrs(1, 10, {21}); } -struct StructMapFixture : public StructFixtureBase -{ - using StructFixtureBase::makeDoc; +class StructMapWriterTest : public StructWriterTestBase { +public: + using StructWriterTestBase::makeDoc; const MapDataType _structMapFieldType; const Field _structMapField; - StructMapFixture() - : StructFixtureBase(), + StructMapWriterTest() + : StructWriterTestBase(), _structMapFieldType(*DataType::INT, _structFieldType), _structMapField("map", _structMapFieldType, true) { - addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); - addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); + addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}); _type.addField(_structMapField); } - std::unique_ptr<Document> - makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) - { + std::unique_ptr<Document> makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) { auto doc = makeDoc(); doc->setValue(_valueField, IntFieldValue(value)); MapFieldValue s(_structMapFieldType); @@ -917,38 +1075,35 @@ struct StructMapFixture : public StructFixtureBase doc->setValue(_structMapField, s); return doc; } + void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) { - auto valueAttr = _m->getAttribute("value")->getSP(); - auto mapKeyAttr = _m->getAttribute("map.key")->getSP(); - auto mapValueAttr = _m->getAttribute("map.value.value")->getSP(); - EXPECT_EQUAL(expValue, valueAttr->getInt(lid)); + auto valueAttr = _mgr->getAttribute("value")->getSP(); + auto mapKeyAttr = _mgr->getAttribute("map.key")->getSP(); + auto mapValueAttr = _mgr->getAttribute("map.value.value")->getSP(); + EXPECT_EQ(expValue, valueAttr->getInt(lid)); attribute::IntegerContent mapKeys; mapKeys.fill(*mapKeyAttr, lid); attribute::IntegerContent mapValues; mapValues.fill(*mapValueAttr, lid); - EXPECT_EQUAL(expMap.size(), mapValues.size()); - EXPECT_EQUAL(expMap.size(), mapKeys.size()); + EXPECT_EQ(expMap.size(), mapValues.size()); + EXPECT_EQ(expMap.size(), mapKeys.size()); size_t i = 0; for (const auto &expMapElem : expMap) { - EXPECT_EQUAL(expMapElem.first, mapKeys[i]); - EXPECT_EQUAL(expMapElem.second, mapValues[i]); + EXPECT_EQ(expMapElem.first, mapKeys[i]); + EXPECT_EQ(expMapElem.second, mapValues[i]); ++i; } } }; -TEST_F("require that update with doc argument updates struct field attributes (map)", StructMapFixture) +TEST_F(StructMapWriterTest, update_with_doc_argument_updates_struct_field_attributes) { - auto doc = f.makeDoc(10, {{1, 11}, {2, 12}}); - f.put(10, *doc, 1); - TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}})); - doc = f.makeDoc(20, {{42, 21}}); - f.update(11, *doc, 1, true); - TEST_DO(f.checkAttrs(1, 10, {{42, 21}})); + auto doc = makeDoc(10, {{1, 11}, {2, 12}}); + put(10, *doc, 1); + checkAttrs(1, 10, {{1, 11}, {2, 12}}); + doc = makeDoc(20, {{42, 21}}); + update(11, *doc, 1, true); + checkAttrs(1, 10, {{42, 21}}); } -TEST_MAIN() -{ - vespalib::rmdir(test_dir, true); - TEST_RUN_ALL(); -} +GTEST_MAIN_RUN_ALL_TESTS() |