diff options
Diffstat (limited to 'searchcore/src')
28 files changed, 757 insertions, 573 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index ee5f29255fb..634f69a3820 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -1,6 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_test"); #include <vespa/config-attributes.h> #include <vespa/document/fieldvalue/document.h> @@ -15,6 +13,7 @@ LOG_SETUP("attribute_test"); #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> @@ -44,6 +43,9 @@ LOG_SETUP("attribute_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/log/log.h> +LOG_SETUP("attribute_test"); + namespace vespa { namespace config { namespace search {}}} using namespace config; @@ -139,6 +141,7 @@ struct Fixture : Fixture(1) { } + ~Fixture(); void allocAttributeWriter() { _aw = std::make_unique<AttributeWriter>(_m); } @@ -155,8 +158,12 @@ struct Fixture _aw->put(serialNum, doc, lid, immediateCommit, emptyCallback); } void update(SerialNum serialNum, const DocumentUpdate &upd, + DocumentIdT lid, bool immediateCommit, IFieldUpdateCallback & onUpdate) { + _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback, onUpdate); + } + void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit) { - _aw->update(serialNum, upd, lid, immediateCommit, emptyCallback); + _aw->update(serialNum, doc, lid, immediateCommit, emptyCallback); } void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit = true) { _aw->remove(serialNum, lid, immediateCommit, emptyCallback); @@ -172,6 +179,7 @@ struct Fixture } }; +Fixture::~Fixture() = default; TEST_F("require that attribute writer handles put", Fixture) { @@ -442,8 +450,9 @@ TEST_F("require that attribute writer handles update", Fixture) upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10))); + DummyFieldUpdateCallback onUpdate; bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + f.update(2, upd, 1, immediateCommit, onUpdate); attribute::IntegerContent ibuf; ibuf.fill(*a1, 1); @@ -453,9 +462,9 @@ TEST_F("require that attribute writer handles update", Fixture) EXPECT_EQUAL(1u, ibuf.size()); EXPECT_EQUAL(30u, ibuf[0]); - f.update(2, upd, 1, immediateCommit); // same sync token as previous + f.update(2, upd, 1, immediateCommit, onUpdate); // same sync token as previous try { - f.update(1, upd, 1, immediateCommit); // lower sync token than previous + f.update(1, upd, 1, immediateCommit, onUpdate); // lower sync token than previous EXPECT_TRUE(true); // update is ignored } catch (vespalib::IllegalStateException & e) { LOG(info, "Got expected exception: '%s'", e.getMessage().c_str()); @@ -488,7 +497,8 @@ TEST_F("require that attribute writer handles predicate update", Fixture) EXPECT_EQUAL(1u, index.getZeroConstraintDocs().size()); EXPECT_FALSE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + DummyFieldUpdateCallback onUpdate; + f.update(2, upd, 1, immediateCommit, onUpdate); EXPECT_EQUAL(0u, index.getZeroConstraintDocs().size()); EXPECT_TRUE(index.getIntervalIndex().lookup(PredicateHash::hash64("foo=bar")).valid()); } @@ -675,7 +685,8 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); bool immediateCommit = true; - f.update(2, upd, 1, immediateCommit); + DummyFieldUpdateCallback onUpdate; + f.update(2, upd, 1, immediateCommit, onUpdate); EXPECT_EQUAL(2u, a1->getNumDocs()); EXPECT_TRUE(tensorAttribute != nullptr); tensor2 = tensorAttribute->getTensor(1); @@ -773,6 +784,158 @@ TEST_F("require that AttributeWriter::forceCommit() clears search cache in impor EXPECT_EQUAL(0u, f._m->getImportedAttributes()->get("imported_b")->getSearchCache()->size()); } +struct StructFixtureBase : public Fixture +{ + DocumentType _type; + const Field _valueField; + StructDataType _structFieldType; + + StructFixtureBase() + : Fixture(), + _type("test"), + _valueField("value", 2, *DataType::INT, true), + _structFieldType("struct") + { + addAttribute({"value", AVConfig(AVBasicType::INT32, AVCollectionType::SINGLE)}, createSerialNum); + _type.addField(_valueField); + _structFieldType.addField(_valueField); + } + ~StructFixtureBase(); + + std::unique_ptr<StructFieldValue> + makeStruct() + { + return std::make_unique<StructFieldValue>(_structFieldType); + } + + std::unique_ptr<StructFieldValue> + makeStruct(const int32_t value) + { + auto ret = makeStruct(); + ret->setValue(_valueField, IntFieldValue(value)); + return ret; + } + + std::unique_ptr<Document> + makeDoc() + { + return std::make_unique<Document>(_type, DocumentId("id::test::1")); + } +}; + +StructFixtureBase::~StructFixtureBase() = default; + +struct StructArrayFixture : public StructFixtureBase +{ + using StructFixtureBase::makeDoc; + const ArrayDataType _structArrayFieldType; + const Field _structArrayField; + + StructArrayFixture() + : StructFixtureBase(), + _structArrayFieldType(_structFieldType), + _structArrayField("array", _structArrayFieldType, true) + { + addAttribute({"array.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + _type.addField(_structArrayField); + } + ~StructArrayFixture(); + + std::unique_ptr<Document> + makeDoc(int32_t value, const std::vector<int32_t> &arrayValues) + { + auto doc = makeDoc(); + doc->setValue(_valueField, IntFieldValue(value)); + ArrayFieldValue s(_structArrayFieldType); + for (const auto &arrayValue : arrayValues) { + s.add(*makeStruct(arrayValue)); + } + doc->setValue(_structArrayField, s); + return doc; + } + void checkAttrs(uint32_t lid, int32_t value, const std::vector<int32_t> &arrayValues) { + auto valueAttr = _m->getAttribute("value")->getSP(); + auto arrayValueAttr = _m->getAttribute("array.value")->getSP(); + EXPECT_EQUAL(value, valueAttr->getInt(lid)); + attribute::IntegerContent ibuf; + ibuf.fill(*arrayValueAttr, lid); + EXPECT_EQUAL(arrayValues.size(), ibuf.size()); + for (size_t i = 0; i < arrayValues.size(); ++i) { + EXPECT_EQUAL(arrayValues[i], ibuf[i]); + } + } +}; + +StructArrayFixture::~StructArrayFixture() = default; + +TEST_F("require that update with doc argument updates compound attributes (array)", StructArrayFixture) +{ + auto doc = f.makeDoc(10, {11, 12}); + f.put(10, *doc, 1); + TEST_DO(f.checkAttrs(1, 10, {11, 12})); + doc = f.makeDoc(20, {21}); + f.update(11, *doc, 1, true); + TEST_DO(f.checkAttrs(1, 10, {21})); +} + +struct StructMapFixture : public StructFixtureBase +{ + using StructFixtureBase::makeDoc; + const MapDataType _structMapFieldType; + const Field _structMapField; + + StructMapFixture() + : StructFixtureBase(), + _structMapFieldType(*DataType::INT, _structFieldType), + _structMapField("map", _structMapFieldType, true) + { + addAttribute({"map.value.value", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + addAttribute({"map.key", AVConfig(AVBasicType::INT32, AVCollectionType::ARRAY)}, createSerialNum); + _type.addField(_structMapField); + } + + std::unique_ptr<Document> + makeDoc(int32_t value, const std::map<int32_t, int32_t> &mapValues) + { + auto doc = makeDoc(); + doc->setValue(_valueField, IntFieldValue(value)); + MapFieldValue s(_structMapFieldType); + for (const auto &mapValue : mapValues) { + s.put(IntFieldValue(mapValue.first), *makeStruct(mapValue.second)); + } + doc->setValue(_structMapField, s); + return doc; + } + void checkAttrs(uint32_t lid, int32_t expValue, const std::map<int32_t, int32_t> &expMap) { + auto valueAttr = _m->getAttribute("value")->getSP(); + auto mapKeyAttr = _m->getAttribute("map.key")->getSP(); + auto mapValueAttr = _m->getAttribute("map.value.value")->getSP(); + EXPECT_EQUAL(expValue, valueAttr->getInt(lid)); + attribute::IntegerContent mapKeys; + mapKeys.fill(*mapKeyAttr, lid); + attribute::IntegerContent mapValues; + mapValues.fill(*mapValueAttr, lid); + EXPECT_EQUAL(expMap.size(), mapValues.size()); + EXPECT_EQUAL(expMap.size(), mapKeys.size()); + size_t i = 0; + for (const auto &expMapElem : expMap) { + EXPECT_EQUAL(expMapElem.first, mapKeys[i]); + EXPECT_EQUAL(expMapElem.second, mapValues[i]); + ++i; + } + } +}; + +TEST_F("require that update with doc argument updates compound attributes (map)", StructMapFixture) +{ + auto doc = f.makeDoc(10, {{1, 11}, {2, 12}}); + f.put(10, *doc, 1); + TEST_DO(f.checkAttrs(1, 10, {{1, 11}, {2, 12}})); + doc = f.makeDoc(20, {{42, 21}}); + f.update(11, *doc, 1, true); + TEST_DO(f.checkAttrs(1, 10, {{42, 21}})); +} + TEST_MAIN() { vespalib::rmdir(test_dir, true); diff --git a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp index d13e4207a9d..bad27938d4b 100644 --- a/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_extractor/document_field_extractor_test.cpp @@ -86,19 +86,6 @@ makeStringWeightedSet(const std::vector<std::pair<vespalib::string, int32_t>> &a return result; } -template <typename F1, typename F2> -void -checkFieldPathChange(F1 f1, F2 f2, const vespalib::string &path, bool same) -{ - FieldPath fieldPath1 = f1.makeFieldPath(path); - FieldPath fieldPath2 = f2.makeFieldPath(path); - EXPECT_TRUE(!fieldPath1.empty()); - EXPECT_TRUE(!fieldPath2.empty()); - EXPECT_TRUE(DocumentFieldExtractor::isSupported(fieldPath1)); - EXPECT_TRUE(DocumentFieldExtractor::isSupported(fieldPath2)); - EXPECT_EQUAL(same, DocumentFieldExtractor::isCompatible(fieldPath1, fieldPath2)); -} - } struct FixtureBase @@ -354,26 +341,6 @@ TEST_F("require that unknown field gives null value", FixtureBase(false)) TEST_DO(f.assertExtracted("unknown", std::unique_ptr<FieldValue>())); } -TEST("require that type changes are detected") -{ - TEST_DO(checkFieldPathChange(SimpleFixture(false), SimpleFixture(false), "weight", true)); - TEST_DO(checkFieldPathChange(SimpleFixture(false), SimpleFixture(true), "weight", false)); - TEST_DO(checkFieldPathChange(ArrayFixture(false), ArrayFixture(false), "weight", true)); - TEST_DO(checkFieldPathChange(ArrayFixture(false), ArrayFixture(true), "weight", false)); - TEST_DO(checkFieldPathChange(SimpleFixture(false), ArrayFixture(false), "weight", false)); - TEST_DO(checkFieldPathChange(WeightedSetFixture(false), WeightedSetFixture(false), "weight", true)); - TEST_DO(checkFieldPathChange(WeightedSetFixture(false), WeightedSetFixture(true), "weight", false)); - TEST_DO(checkFieldPathChange(SimpleFixture(false), WeightedSetFixture(false), "weight", false)); - TEST_DO(checkFieldPathChange(ArrayFixture(false), WeightedSetFixture(false), "weight", false)); - TEST_DO(checkFieldPathChange(StructArrayFixture(false), StructArrayFixture(false), "s.weight", true)); - TEST_DO(checkFieldPathChange(StructArrayFixture(false), StructArrayFixture(true), "s.weight", false)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, false), "s.value.weight", true)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(true, false), "s.value.weight", false)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, true), "s.value.weight", false)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, false), "s.key", true)); - TEST_DO(checkFieldPathChange(StructMapFixture(false, false), StructMapFixture(false, true), "s.key", false)); -} - TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 00eb59f120a..5d040024e63 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcore/proton/attribute/i_attribute_writer.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/searchcore/proton/common/feedtoken.h> @@ -316,21 +317,23 @@ struct MyAttributeWriter : public IAttributeWriter std::set<vespalib::string> _attrs; proton::IAttributeManager::SP _mgr; MyTracer &_tracer; + MyAttributeWriter(MyTracer &tracer); ~MyAttributeWriter(); - virtual std::vector<AttributeVector *> + + std::vector<AttributeVector *> getWritableAttributes() const override { return std::vector<AttributeVector *>(); } - virtual AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override { + AttributeVector *getWritableAttribute(const vespalib::string &attrName) const override { if (_attrs.count(attrName) == 0) { return nullptr; } AttrMap::const_iterator itr = _attrMap.find(attrName); return ((itr == _attrMap.end()) ? nullptr : itr->second.get()); } - virtual void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType) override { + void put(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { _putSerial = serialNum; _putDocId = doc.getId(); _putLid = lid; @@ -339,8 +342,8 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; } } - virtual void remove(SerialNum serialNum, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType) override { + void remove(SerialNum serialNum, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { _removeSerial = serialNum; _removeLid = lid; _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit); @@ -348,37 +351,45 @@ struct MyAttributeWriter : public IAttributeWriter ++_commitCount; } } - virtual void remove(const LidVector & lidsToRemove, SerialNum serialNum, - bool immediateCommit, OnWriteDoneType) override { + void remove(const LidVector & lidsToRemove, SerialNum serialNum, + bool immediateCommit, OnWriteDoneType) override { for (uint32_t lid : lidsToRemove) { LOG(info, "MyAttributeAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid); _removes.push_back(lid); _tracer.traceRemove(attributeAdapterTypeName, serialNum, lid, immediateCommit); } } - virtual void update(SerialNum serialNum, const document::DocumentUpdate &upd, - DocumentIdT lid, bool, OnWriteDoneType) override { + void update(SerialNum serialNum, const document::DocumentUpdate &upd, + DocumentIdT lid, bool, OnWriteDoneType, IFieldUpdateCallback & onUpdate) override { _updateSerial = serialNum; _updateDocId = upd.getId(); _updateLid = lid; + for (const auto & fieldUpdate : upd.getUpdates()) { + search::AttributeVector * attr = getWritableAttribute(fieldUpdate.getField().getName()); + onUpdate.onUpdateField(fieldUpdate.getField().getName(), attr); + } } - virtual void heartBeat(SerialNum) override { ++_heartBeatCount; } - virtual void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override { + void update(SerialNum serialNum, const document::Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType) override { (void) serialNum; + (void) doc; + (void) lid; + (void) immediateCommit; + } + void heartBeat(SerialNum) override { ++_heartBeatCount; } + void compactLidSpace(uint32_t wantedLidLimit, SerialNum ) override { _wantedLidLimit = wantedLidLimit; } - virtual const proton::IAttributeManager::SP &getAttributeManager() const override { + const proton::IAttributeManager::SP &getAttributeManager() const override { return _mgr; } void forceCommit(SerialNum serialNum, OnWriteDoneType) override { - (void) serialNum; ++_commitCount; + ++_commitCount; _tracer.traceCommit(attributeAdapterTypeName, serialNum); } - virtual void onReplayDone(uint32_t docIdLimit) override - { - (void) docIdLimit; - } + void onReplayDone(uint32_t ) override { } + bool getHasCompoundAttribute() const override { return false; } }; MyAttributeWriter::MyAttributeWriter(MyTracer &tracer) @@ -396,7 +407,7 @@ MyAttributeWriter::MyAttributeWriter(MyTracer &tracer) cfg3.setTensorType(ValueType::from_spec("tensor(x[10])")); _attrMap["a3"] = search::AttributeFactory::createAttribute("test3", cfg3); } -MyAttributeWriter::~MyAttributeWriter() {} +MyAttributeWriter::~MyAttributeWriter() = default; struct MyTransport : public feedtoken::ITransport { @@ -420,7 +431,7 @@ struct MyResultHandler : public IGenericResultHandler { vespalib::Gate _gate; MyResultHandler() : _gate() {} - virtual void handle(const storage::spi::Result &) override { + void handle(const storage::spi::Result &) override { _gate.countDown(); } void await() { _gate.await(); } @@ -446,7 +457,7 @@ SchemaContext::SchemaContext() : _schema->addSummaryField(Schema::SummaryField("s1", DataType::STRING, CollectionType::SINGLE)); _builder.reset(new DocBuilder(*_schema)); } -SchemaContext::~SchemaContext() {} +SchemaContext::~SchemaContext() = default; struct DocumentContext diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index a6d7f12f199..681d3543ee1 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -23,6 +23,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> @@ -282,8 +283,10 @@ TEST_F("require that we can deserialize old update operations", Fixture) BucketId bucket(toBucket(docId.getGlobalId())); auto upd(f.makeUpdate()); { - UpdateOperation op(UpdateOperation::makeOldUpdate(bucket, Timestamp(10), upd)); - op.serialize(stream); + UpdateOperation op(bucket, Timestamp(10), upd); + op.serializeDocumentOperationOnly(stream); + document::VespaDocumentSerializer serializer(stream); + serializer.write42(*op.getUpdate()); } { UpdateOperation op(FeedOperation::UPDATE_42); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 62c61406f67..a770fff3f5f 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -61,6 +61,36 @@ void inject_match_phase_limiting(Properties &setup, const vespalib::string &attr setup.import(cfg); } +FakeResult make_elem_result(const std::vector<std::pair<uint32_t,std::vector<uint32_t> > > &match_data) { + FakeResult result; + uint32_t pos_should_be_ignored = 0; + for (const auto &doc: match_data) { + result.doc(doc.first); + for (const auto &elem: doc.second) { + result.elem(elem).pos(++pos_should_be_ignored); + } + } + return result; +} + +vespalib::string make_simple_stack_dump(const vespalib::string &field, + const vespalib::string &term) +{ + QueryBuilder<ProtonNodeTypes> builder; + builder.addStringTerm(term, field, 1, search::query::Weight(1)); + return StackDumpCreator::create(*builder.build()); +} + +vespalib::string make_same_element_stack_dump(const vespalib::string &a1_term, + const vespalib::string &f1_term) +{ + QueryBuilder<ProtonNodeTypes> builder; + builder.addSameElement(2, "ignored field name"); + builder.addStringTerm(a1_term, "a1", 1, search::query::Weight(1)); + builder.addStringTerm(f1_term, "f1", 2, search::query::Weight(1)); + return StackDumpCreator::create(*builder.build()); +} + //----------------------------------------------------------------------------- const uint32_t NUM_DOCS = 1000; @@ -238,6 +268,13 @@ struct MyWorld { searchContext.attr().addResult("a1", term, result); } + void add_same_element_results(const vespalib::string &a1_term, const vespalib::string &f1_0_term) { + auto a1_result = make_elem_result({{10, {1}}, {20, {2}}, {21, {2}}}); + auto f1_0_result = make_elem_result({{10, {2}}, {20, {2}}, {21, {2}}}); + searchContext.attr().addResult("a1", a1_term, a1_result); + searchContext.idx(0).getFake().addResult("f1", f1_0_term, f1_0_result); + } + void basicResults() { searchContext.idx(0).getFake().addResult("f1", "foo", FakeResult() @@ -249,26 +286,32 @@ struct MyWorld { .doc(600).doc(700).doc(800).doc(900)); } - void setStackDump(Request &request, const vespalib::string &field, - const vespalib::string &term) { - QueryBuilder<ProtonNodeTypes> builder; - builder.addStringTerm(term, field, 1, search::query::Weight(1)); - vespalib::string stack_dump = - StackDumpCreator::create(*builder.build()); + void setStackDump(Request &request, const vespalib::string &stack_dump) { request.stackDump.assign(stack_dump.data(), stack_dump.data() + stack_dump.size()); } - SearchRequest::SP createSimpleRequest(const vespalib::string &field, - const vespalib::string &term) + SearchRequest::SP createRequest(const vespalib::string &stack_dump) { SearchRequest::SP request(new SearchRequest); request->setTimeout(60 * fastos::TimeStamp::SEC); - setStackDump(*request, field, term); + setStackDump(*request, stack_dump); request->maxhits = 10; return request; } + SearchRequest::SP createSimpleRequest(const vespalib::string &field, + const vespalib::string &term) + { + return createRequest(make_simple_stack_dump(field, term)); + } + + SearchRequest::SP createSameElementRequest(const vespalib::string &a1_term, + const vespalib::string &f1_term) + { + return createRequest(make_same_element_stack_dump(a1_term, f1_term)); + } + Matcher::SP createMatcher() { return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, 0); } @@ -317,7 +360,7 @@ struct MyWorld { const vespalib::string & term) { DocsumRequest::SP request(new DocsumRequest); - setStackDump(*request, field, term); + setStackDump(*request, make_simple_stack_dump(field, term)); // match a subset of basic result + request for a non-hit (not // sorted on docid) @@ -800,4 +843,14 @@ TEST("require that fields are tagged with data type") { EXPECT_EQUAL(predicate_field->get_data_type(), FieldInfo::DataType::BOOLEANTREE); } +TEST("require that same element search works (note that this does not test/use the attribute element iterator wrapper)") { + MyWorld world; + world.basicSetup(); + world.add_same_element_results("foo", "bar"); + SearchRequest::SP request = world.createSameElementRequest("foo", "bar"); + SearchReply::UP reply = world.performSearch(request, 1); + ASSERT_EQUAL(1u, reply->hits.size()); + EXPECT_EQUAL(document::DocumentId("doc::20").getGlobalId(), reply->hits[0].gid); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 9262f9a7b6f..705a27c7fc3 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -68,24 +68,8 @@ createUpd(const DocumentType& docType, const DocumentId &docId) return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId)); } - -document::Document::UP -clone(const document::Document::SP &doc) -{ - return document::Document::UP(doc->clone()); -} - - -document::DocumentUpdate::UP -clone(const document::DocumentUpdate::SP &upd) -{ - return document::DocumentUpdate::UP(upd->clone()); -} - - storage::spi::ClusterState -createClusterState(const storage::lib::State& nodeState = - storage::lib::State::UP) +createClusterState(const storage::lib::State& nodeState = storage::lib::State::UP) { using storage::lib::Distribution; using storage::lib::Node; @@ -99,11 +83,7 @@ createClusterState(const storage::lib::State& nodeState = StorDistributionConfigBuilder dc; cstate.setNodeState(Node(NodeType::STORAGE, 0), - NodeState(NodeType::STORAGE, - nodeState, - "dummy desc", - 1.0, - 1)); + NodeState(NodeType::STORAGE, nodeState, "dummy desc", 1.0, 1)); cstate.setClusterState(State::UP); dc.redundancy = 1; dc.readyCopies = 1; @@ -222,8 +202,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { void handleUpdate(FeedToken token, const Bucket& bucket, Timestamp timestamp, const document::DocumentUpdate::SP& upd) override { - token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), - existingTimestamp > 0); + token->setResult(ResultUP(new storage::spi::UpdateResult(existingTimestamp)), existingTimestamp > 0); handle(token, bucket, timestamp, upd->getId()); } @@ -312,8 +291,7 @@ struct MyHandler : public IPersistenceHandler, IBucketFreezer { return frozen.find(bucket.getBucketId().getId()) != frozen.end(); } bool wasFrozen(const Bucket &bucket) { - return was_frozen.find(bucket.getBucketId().getId()) - != was_frozen.end(); + return was_frozen.find(bucket.getBucketId().getId()) != was_frozen.end(); } }; @@ -335,7 +313,7 @@ HandlerSet::HandlerSet() handler1(static_cast<MyHandler &>(*phandler1.get())), handler2(static_cast<MyHandler &>(*phandler2.get())) {} -HandlerSet::~HandlerSet() {} +HandlerSet::~HandlerSet() = default; DocumentType type1(createDocType("type1", 1)); DocumentType type2(createDocType("type2", 2)); @@ -405,8 +383,8 @@ struct SimpleResourceWriteFilter : public IResourceWriteFilter _message() {} - virtual bool acceptWriteOperation() const override { return _acceptWriteOperation; } - virtual State getAcceptState() const override { + bool acceptWriteOperation() const override { return _acceptWriteOperation; } + State getAcceptState() const override { return IResourceWriteFilter::State(acceptWriteOperation(), _message); } }; @@ -475,8 +453,7 @@ TEST_F("require that getPartitionStates() prepares all handlers", SimpleFixture) TEST_F("require that puts are routed to handler", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.put(bucket1, tstamp1, doc1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); @@ -485,20 +462,16 @@ TEST_F("require that puts are routed to handler", SimpleFixture) assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), - f.engine.put(bucket1, tstamp1, doc3, context)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), + f.engine.put(bucket1, tstamp1, doc3, context)); } TEST_F("require that puts with old id scheme are rejected", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "Old id scheme not supported in " - "elastic mode (doc:old:id-scheme)"), - f.engine.put(bucket1, tstamp1, old_doc, context)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "Old id scheme not supported in elastic mode (doc:old:id-scheme)"), + f.engine.put(bucket1, tstamp1, old_doc, context)); } @@ -508,8 +481,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::RESOURCE_EXHAUSTED, "Put operation rejected for document 'doc:old:id-scheme': 'Disk is full'"), @@ -520,8 +492,7 @@ TEST_F("require that put is rejected if resource limit is reached", SimpleFixtur TEST_F("require that updates are routed to handler", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.setExistingTimestamp(tstamp2); UpdateResult ur = f.engine.update(bucket1, tstamp1, upd1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); @@ -534,9 +505,8 @@ TEST_F("require that updates are routed to handler", SimpleFixture) assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); EXPECT_EQUAL(tstamp3, ur.getExistingTimestamp()); - EXPECT_EQUAL( - Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), - f.engine.update(bucket1, tstamp1, upd3, context)); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), + f.engine.update(bucket1, tstamp1, upd3, context)); } @@ -546,8 +516,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL( Result(Result::RESOURCE_EXHAUSTED, @@ -559,8 +528,7 @@ TEST_F("require that update is rejected if resource limit is reached", SimpleFix TEST_F("require that removes are routed to handlers", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); RemoveResult rr = f.engine.remove(bucket1, tstamp1, docId3, context); assertHandler(bucket0, tstamp0, docId0, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); @@ -598,8 +566,7 @@ TEST_F("require that remove is NOT rejected if resource limit is reached", Simpl f._writeFilter._message = "Disk is full"; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); EXPECT_EQUAL(RemoveResult(false), f.engine.remove(bucket1, tstamp1, docId1, context)); } @@ -628,8 +595,7 @@ TEST_F("require that setActiveState() is routed to handlers and merged", SimpleF f.hset.handler1.bucketStateResult = Result(Result::TRANSIENT_ERROR, "err1"); f.hset.handler2.bucketStateResult = Result(Result::PERMANENT_ERROR, "err2"); - Result result = f.engine.setActiveState(bucket1, - storage::spi::BucketInfo::NOT_ACTIVE); + Result result = f.engine.setActiveState(bucket1, storage::spi::BucketInfo::NOT_ACTIVE); EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode()); EXPECT_EQUAL("err1, err2", result.getErrorMessage()); EXPECT_EQUAL(storage::spi::BucketInfo::NOT_ACTIVE, f.hset.handler1.lastBucketState); @@ -651,16 +617,12 @@ TEST_F("require that getBucketInfo() is routed to handlers and merged", SimpleFi } -TEST_F("require that createBucket() is routed to handlers and merged", - SimpleFixture) +TEST_F("require that createBucket() is routed to handlers and merged", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - f.hset.handler1._createBucketResult = - Result(Result::TRANSIENT_ERROR, "err1a"); - f.hset.handler2._createBucketResult = - Result(Result::PERMANENT_ERROR, "err2a"); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + f.hset.handler1._createBucketResult = Result(Result::TRANSIENT_ERROR, "err1a"); + f.hset.handler2._createBucketResult = Result(Result::PERMANENT_ERROR, "err2a"); Result result = f.engine.createBucket(bucket1, context); EXPECT_EQUAL(Result::PERMANENT_ERROR, result.getErrorCode()); @@ -671,8 +633,7 @@ TEST_F("require that createBucket() is routed to handlers and merged", TEST_F("require that deleteBucket() is routed to handlers and merged", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.hset.handler1.deleteBucketResult = Result(Result::TRANSIENT_ERROR, "err1"); f.hset.handler2.deleteBucketResult = Result(Result::PERMANENT_ERROR, "err2"); @@ -691,10 +652,8 @@ TEST_F("require that getModifiedBuckets() is routed to handlers and merged", Sim TEST_F("require that get is sent to all handlers", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); EXPECT_EQUAL(docId1, f.hset.handler2.lastDocId); @@ -704,8 +663,7 @@ TEST_F("require that get freezes the bucket", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.wasFrozen(bucket1)); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_TRUE(f.hset.handler1.wasFrozen(bucket1)); EXPECT_TRUE(f.hset.handler2.wasFrozen(bucket1)); @@ -717,10 +675,8 @@ TEST_F("require that get returns the first document found", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); f.hset.handler2.setDocument(*doc2, tstamp2); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + GetResult result = f.engine.get(bucket1, document::AllFields(), docId1, context); EXPECT_EQUAL(docId1, f.hset.handler1.lastDocId); EXPECT_EQUAL(DocumentId(), f.hset.handler2.lastDocId); @@ -732,8 +688,7 @@ TEST_F("require that get returns the first document found", SimpleFixture) { TEST_F("require that createIterator does", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -741,15 +696,13 @@ TEST_F("require that createIterator does", SimpleFixture) { EXPECT_TRUE(result.getIteratorId()); uint64_t max_size = 1024; - IterateResult it_result = - f.engine.iterate(result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context); EXPECT_FALSE(it_result.hasError()); } TEST_F("require that iterator ids are unique", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); @@ -764,10 +717,8 @@ TEST_F("require that iterator ids are unique", SimpleFixture) { TEST_F("require that iterate requires valid iterator", SimpleFixture) { uint64_t max_size = 1024; storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); - IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, - context); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); + IterateResult it_result = f.engine.iterate(IteratorId(1), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode()); EXPECT_EQUAL("Unknown iterator with id 1", it_result.getErrorMessage()); @@ -786,16 +737,14 @@ TEST_F("require that iterate returns documents", SimpleFixture) { f.hset.handler2.setDocument(*doc2, tstamp2); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); uint64_t max_size = 1024; CreateIteratorResult result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(result.getIteratorId()); - IterateResult it_result = - f.engine.iterate(result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(result.getIteratorId(), max_size, context); EXPECT_FALSE(it_result.hasError()); EXPECT_EQUAL(2u, it_result.getEntries().size()); } @@ -804,33 +753,28 @@ TEST_F("require that destroyIterator prevents iteration", SimpleFixture) { f.hset.handler1.setDocument(*doc1, tstamp1); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(create_result.getIteratorId()); - Result result = f.engine.destroyIterator(create_result.getIteratorId(), - context); + Result result = f.engine.destroyIterator(create_result.getIteratorId(), context); EXPECT_FALSE(result.hasError()); uint64_t max_size = 1024; - IterateResult it_result = - f.engine.iterate(create_result.getIteratorId(), max_size, context); + IterateResult it_result = f.engine.iterate(create_result.getIteratorId(), max_size, context); EXPECT_TRUE(it_result.hasError()); EXPECT_EQUAL(Result::PERMANENT_ERROR, it_result.getErrorCode()); string msg_prefix = "Unknown iterator with id"; - EXPECT_EQUAL(msg_prefix, - it_result.getErrorMessage().substr(0, msg_prefix.size())); + EXPECT_EQUAL(msg_prefix, it_result.getErrorMessage().substr(0, msg_prefix.size())); } TEST_F("require that buckets are frozen during iterator life", SimpleFixture) { EXPECT_FALSE(f.hset.handler1.isFrozen(bucket1)); EXPECT_FALSE(f.hset.handler2.isFrozen(bucket1)); storage::spi::LoadType loadType(0, "default"); - Context context(loadType, storage::spi::Priority(0), - storage::spi::Trace::TraceLevel(0)); + Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 0cb260ed9a8..35f5f09fc37 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_writer.h" +#include "ifieldupdatecallback.h" #include "attributemanager.h" #include "document_field_extractor.h" #include <vespa/document/base/exceptions.h> @@ -22,10 +23,34 @@ namespace proton { using LidVector = LidVectorContext::LidVector; +AttributeWriter::WriteField::WriteField(AttributeVector &attribute) + : _fieldPath(), + _attribute(attribute), + _compoundAttribute(false) +{ + const vespalib::string &name = attribute.getName(); + _compoundAttribute = name.find('.') != vespalib::string::npos; +} + +AttributeWriter::WriteField::~WriteField() = default; + +void +AttributeWriter::WriteField::buildFieldPath(const DocumentType &docType) +{ + const vespalib::string &name = _attribute.getName(); + FieldPath fp; + try { + docType.buildFieldPath(fp, name); + } catch (document::FieldNotFoundException & e) { + fp = FieldPath(); + } + _fieldPath = std::move(fp); +} + AttributeWriter::WriteContext::WriteContext(uint32_t executorId) : _executorId(executorId), - _fieldPaths(), - _attributes() + _fields(), + _hasCompoundAttribute(false) { } @@ -37,30 +62,20 @@ AttributeWriter::WriteContext::~WriteContext() = default; AttributeWriter::WriteContext &AttributeWriter::WriteContext::operator=(WriteContext &&rhs) = default; void -AttributeWriter::WriteContext::add(AttributeVector *attr) +AttributeWriter::WriteContext::add(AttributeVector &attr) { - _attributes.emplace_back(attr); - _fieldPaths.emplace_back(); + _fields.emplace_back(attr); + if (_fields.back().getCompoundAttribute()) { + _hasCompoundAttribute = true; + } } void AttributeWriter::WriteContext::buildFieldPaths(const DocumentType &docType) { - size_t fieldId = 0; - for (const auto &attrp : _attributes) { - const vespalib::string &name = attrp->getName(); - FieldPath fp; - try { - docType.buildFieldPath(fp, name); - } catch (document::FieldNotFoundException & e) { - fp = FieldPath(); - } - - assert(fieldId < _fieldPaths.size()); - _fieldPaths[fieldId] = std::move(fp); - ++fieldId; + for (auto &field : _fields) { + field.buildFieldPath(docType); } - assert(fieldId == _fieldPaths.size()); } namespace { @@ -200,26 +215,30 @@ class PutTask : public vespalib::Executor::Task const SerialNum _serialNum; const uint32_t _lid; const bool _immediateCommit; + const bool _allAttributes; std::remove_reference_t<AttributeWriter::OnWriteDoneType> _onWriteDone; std::vector<FieldValue::UP> _fieldValues; public: - PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone); + PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone); virtual ~PutTask() override; virtual void run() override; }; -PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, AttributeWriter::OnWriteDoneType onWriteDone) +PutTask::PutTask(const AttributeWriter::WriteContext &wc, SerialNum serialNum, DocumentFieldExtractor &fieldExtractor, uint32_t lid, bool immediateCommit, bool allAttributes, AttributeWriter::OnWriteDoneType onWriteDone) : _wc(wc), _serialNum(serialNum), _lid(lid), _immediateCommit(immediateCommit), + _allAttributes(allAttributes), _onWriteDone(onWriteDone) { - const auto &fieldPaths = _wc.getFieldPaths(); - _fieldValues.reserve(fieldPaths.size()); - for (const auto &fieldPath : fieldPaths) { - FieldValue::UP fv = fieldExtractor.getFieldValue(fieldPath); - _fieldValues.emplace_back(std::move(fv)); + const auto &fields = _wc.getFields(); + _fieldValues.reserve(fields.size()); + for (const auto &field : fields) { + if (_allAttributes || field.getCompoundAttribute()) { + FieldValue::UP fv = fieldExtractor.getFieldValue(field.getFieldPath()); + _fieldValues.emplace_back(std::move(fv)); + } } } @@ -231,13 +250,15 @@ void PutTask::run() { uint32_t fieldId = 0; - const auto &attributes = _wc.getAttributes(); - for (auto attrp : attributes) { - AttributeVector &attr = *attrp; - if (attr.getStatus().getLastSyncToken() < _serialNum) { - applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone); + const auto &fields = _wc.getFields(); + for (auto field : fields) { + if (_allAttributes || field.getCompoundAttribute()) { + AttributeVector &attr = field.getAttribute(); + if (attr.getStatus().getLastSyncToken() < _serialNum) { + applyPutToAttribute(_serialNum, _fieldValues[fieldId], _lid, _immediateCommit, attr, _onWriteDone); + } + ++fieldId; } - ++fieldId; } } @@ -271,9 +292,9 @@ RemoveTask::~RemoveTask() void RemoveTask::run() { - const auto &attributes = _wc.getAttributes(); - for (auto &attrp : attributes) { - AttributeVector &attr = *attrp; + const auto &fields = _wc.getFields(); + for (auto &field : fields) { + AttributeVector &attr = field.getAttribute(); // Must use <= due to how move operations are handled if (attr.getStatus().getLastSyncToken() <= _serialNum) { applyRemoveToAttribute(_serialNum, _lid, _immediateCommit, attr, _onWriteDone); @@ -303,13 +324,14 @@ public: {} virtual ~BatchRemoveTask() override {} virtual void run() override { - for (auto attr : _writeCtx.getAttributes()) { - if (attr->getStatus().getLastSyncToken() < _serialNum) { + for (auto field : _writeCtx.getFields()) { + auto &attr = field.getAttribute(); + if (attr.getStatus().getLastSyncToken() < _serialNum) { for (auto lidToRemove : _lidsToRemove) { - applyRemoveToAttribute(_serialNum, lidToRemove, false, *attr, _onWriteDone); + applyRemoveToAttribute(_serialNum, lidToRemove, false, attr, _onWriteDone); } if (_immediateCommit) { - attr->commit(_serialNum, _serialNum); + attr.commit(_serialNum, _serialNum); } } } @@ -342,9 +364,9 @@ CommitTask::~CommitTask() void CommitTask::run() { - const auto &attributes = _wc.getAttributes(); - for (auto &attrp : attributes) { - AttributeVector &attr = *attrp; + const auto &fields = _wc.getFields(); + for (auto &field : fields) { + AttributeVector &attr = field.getAttribute(); applyCommit(_serialNum, _onWriteDone, attr); } } @@ -365,7 +387,12 @@ AttributeWriter::setupWriteContexts() (_writeContexts.back().getExecutorId() != fc.getExecutorId())) { _writeContexts.emplace_back(fc.getExecutorId()); } - _writeContexts.back().add(fc.getAttribute()); + _writeContexts.back().add(*fc.getAttribute()); + } + for (const auto &wc : _writeContexts) { + if (wc.getHasCompoundAttribute()) { + _hasCompoundAttribute = true; + } } } @@ -380,12 +407,18 @@ AttributeWriter::buildFieldPaths(const DocumentType & docType, const DataType *d void AttributeWriter::internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone) + bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone) { + const DataType *dataType(doc.getDataType()); + if (_dataType != dataType) { + buildFieldPaths(doc.getType(), dataType); + } DocumentFieldExtractor extractor(doc); for (const auto &wc : _writeContexts) { - auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, onWriteDone); - _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + if (allAttributes || wc.getHasCompoundAttribute()) { + auto putTask = std::make_unique<PutTask>(wc, serialNum, extractor, lid, immediateCommit, allAttributes, onWriteDone); + _attributeFieldWriter.executeTask(wc.getExecutorId(), std::move(putTask)); + } } } @@ -405,7 +438,8 @@ AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr) _attributeFieldWriter(mgr->getAttributeFieldWriter()), _writableAttributes(mgr->getWritableAttributes()), _writeContexts(), - _dataType(nullptr) + _dataType(nullptr), + _hasCompoundAttribute(false) { setupWriteContexts(); } @@ -432,18 +466,26 @@ void AttributeWriter::put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) { - FieldValue::UP attrVal; LOG(spam, "Handle put: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)", serialNum, doc.getId().toString().c_str(), lid, doc.toString(true).c_str()); - const DataType *dataType(doc.getDataType()); - if (_dataType != dataType) { - buildFieldPaths(doc.getType(), dataType); - } - internalPut(serialNum, doc, lid, immediateCommit, onWriteDone); + internalPut(serialNum, doc, lid, immediateCommit, true, onWriteDone); +} + +void +AttributeWriter::update(SerialNum serialNum, const Document &doc, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone) +{ + LOG(spam, + "Handle update: serial(%" PRIu64 "), docId(%s), lid(%u), document(%s)", + serialNum, + doc.getId().toString().c_str(), + lid, + doc.toString(true).c_str()); + internalPut(serialNum, doc, lid, immediateCommit, false, onWriteDone); } void @@ -465,17 +507,15 @@ AttributeWriter::remove(const LidVector &lidsToRemove, SerialNum serialNum, void AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone) + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) { LOG(debug, "Inspecting update for document %d.", lid); for (const auto &fupd : upd.getUpdates()) { - LOG(debug, "Retrieving guard for attribute vector '%s'.", - fupd.getField().getName().c_str()); - AttributeVector *attrp = - _mgr->getWritableAttribute(fupd.getField().getName()); + LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str()); + AttributeVector *attrp = _mgr->getWritableAttribute(fupd.getField().getName()); + onUpdate.onUpdateField(fupd.getField().getName(), attrp); if (attrp == nullptr) { - LOG(spam, "Failed to find attribute vector %s", - fupd.getField().getName().c_str()); + LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str()); continue; } AttributeVector &attr = *attrp; @@ -484,8 +524,7 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document if (attr.getStatus().getLastSyncToken() >= serialNum) continue; - LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", - lid, attr.getName().c_str()); + LOG(debug, "About to apply update for docId %u in attribute vector '%s'.", lid, attr.getName().c_str()); // NOTE: The lifetime of the field update will be ensured by keeping the document update alive // in a operation done context object. @@ -550,5 +589,11 @@ AttributeWriter::compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) _attributeFieldWriter.sync(); } +bool +AttributeWriter::getHasCompoundAttribute() const +{ + return _hasCompoundAttribute; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h index cbda11180c0..dfdafa3bea9 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h @@ -23,30 +23,44 @@ private: search::ISequencedTaskExecutor &_attributeFieldWriter; const std::vector<search::AttributeVector *> &_writableAttributes; public: + class WriteField + { + FieldPath _fieldPath; + AttributeVector &_attribute; + bool _compoundAttribute; // in array/map of struct + public: + WriteField(AttributeVector &attribute); + ~WriteField(); + AttributeVector &getAttribute() const { return _attribute; } + const FieldPath &getFieldPath() const { return _fieldPath; } + void buildFieldPath(const DocumentType &docType); + bool getCompoundAttribute() const { return _compoundAttribute; } + }; class WriteContext { uint32_t _executorId; - std::vector<FieldPath> _fieldPaths; - std::vector<AttributeVector *> _attributes; + std::vector<WriteField> _fields; + bool _hasCompoundAttribute; public: WriteContext(uint32_t executorId); WriteContext(WriteContext &&rhs); ~WriteContext(); WriteContext &operator=(WriteContext &&rhs); void buildFieldPaths(const DocumentType &docType); - void add(AttributeVector *attr); + void add(AttributeVector &attr); uint32_t getExecutorId() const { return _executorId; } - const std::vector<FieldPath> &getFieldPaths() const { return _fieldPaths; } - const std::vector<AttributeVector *> &getAttributes() const { return _attributes; } + const std::vector<WriteField> &getFields() const { return _fields; } + bool getHasCompoundAttribute() const { return _hasCompoundAttribute; } }; private: std::vector<WriteContext> _writeContexts; const DataType *_dataType; + bool _hasCompoundAttribute; void setupWriteContexts(); void buildFieldPaths(const DocumentType &docType, const DataType *dataType); void internalPut(SerialNum serialNum, const Document &doc, DocumentIdT lid, - bool immediateCommit, OnWriteDoneType onWriteDone); + bool immediateCommit, bool allAttributes, OnWriteDoneType onWriteDone); void internalRemove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone); @@ -68,6 +82,8 @@ public: void remove(const LidVector &lidVector, SerialNum serialNum, bool immediateCommit, OnWriteDoneType onWriteDone) override; void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override; + void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) override; void heartBeat(SerialNum serialNum) override; void compactLidSpace(uint32_t wantedLidLimit, SerialNum serialNum) override; @@ -76,7 +92,8 @@ public: } void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; - virtual void onReplayDone(uint32_t docIdLimit) override; + void onReplayDone(uint32_t docIdLimit) override; + bool getHasCompoundAttribute() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp index 143441eaae9..46f3fdeff67 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp @@ -146,23 +146,6 @@ DocumentFieldExtractor::isSupported(const FieldPath &fieldPath) return true; } -bool -DocumentFieldExtractor::isCompatible(const FieldPath &fieldPath1, const FieldPath &fieldPath2) -{ - if (fieldPath1.size() != fieldPath2.size()) { - return false; - } - uint32_t arrayIndex = 0; - for (const auto &fieldPathEntry1 : fieldPath1) { - const auto &fieldPathEntry2 = fieldPath2[arrayIndex++]; - if (fieldPathEntry1->getType() != fieldPathEntry2.getType() || - fieldPathEntry1->getDataType() != fieldPathEntry2.getDataType()) { - return false; - } - } - return true; -} - const FieldValue * DocumentFieldExtractor::getCachedFieldValue(const FieldPathEntry &fieldPathEntry) { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h index 71f020a582c..48e2da9c4c6 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.h @@ -41,15 +41,6 @@ public: * Check if fieldPath is in a supported form. */ static bool isSupported(const document::FieldPath &fieldPath); - - /** - * Check if two field paths are compatible, i.e. same types in whole path - * and same data type would be returned from getFieldValue(). This is - * meant to be used when document type in received document doesn't match - * the document type for the current config (can happen right before and - * after live config change when validation override is used). - */ - static bool isCompatible(const document::FieldPath &fieldPath1, const document::FieldPath &fieldPath2); }; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h index abcf132d537..7a557b17964 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_writer.h @@ -1,18 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> +#include "i_attribute_manager.h" +#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/query/base.h> #include <vespa/searchlib/common/serialnum.h> -#include <vespa/searchcore/proton/attribute/i_attribute_manager.h> -#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/update/documentupdate.h> namespace search { class IDestructorCallback; } namespace proton { +class IFieldUpdateCallback; + /** * Interface for an attribute writer that handles writes in form of put, update and remove * to an underlying set of attribute vectors. @@ -31,10 +33,8 @@ public: virtual ~IAttributeWriter() {} - virtual std::vector<search::AttributeVector *> - getWritableAttributes() const = 0; - virtual search::AttributeVector * - getWritableAttribute(const vespalib::string &attrName) const = 0; + virtual std::vector<search::AttributeVector *> getWritableAttributes() const = 0; + virtual search::AttributeVector *getWritableAttribute(const vespalib::string &attrName) const = 0; virtual void put(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) = 0; virtual void remove(SerialNum serialNum, DocumentIdT lid, bool immediateCommit, @@ -46,6 +46,11 @@ public: * The OnWriteDoneType instance should ensure the lifetime of the given DocumentUpdate instance. */ virtual void update(SerialNum serialNum, const DocumentUpdate &upd, DocumentIdT lid, + bool immediateCommit, OnWriteDoneType onWriteDone, IFieldUpdateCallback & onUpdate) = 0; + /* + * Update the underlying compound attributes based on updated document. + */ + virtual void update(SerialNum serialNum, const Document &doc, DocumentIdT lid, bool immediateCommit, OnWriteDoneType onWriteDone) = 0; virtual void heartBeat(SerialNum serialNum) = 0; /** @@ -60,6 +65,8 @@ public: virtual void forceCommit(SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void onReplayDone(uint32_t docIdLimit) = 0; + + virtual bool getHasCompoundAttribute() const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h new file mode 100644 index 00000000000..ffb8555cd2c --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/ifieldupdatecallback.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace search { class AttributeVector; } + +namespace proton { + +struct IFieldUpdateCallback { + virtual ~IFieldUpdateCallback() { } + virtual void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) = 0; +}; + +struct DummyFieldUpdateCallback : IFieldUpdateCallback { + void onUpdateField(vespalib::stringref, const search::AttributeVector *) override {} +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp index b2567527560..37b23449315 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp @@ -26,9 +26,7 @@ DocumentOperation::DocumentOperation(Type type) } -DocumentOperation::DocumentOperation(Type type, - const BucketId &bucketId, - const Timestamp ×tamp) +DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const Timestamp ×tamp) : FeedOperation(type), _bucketId(bucketId), _timestamp(timestamp), @@ -62,7 +60,12 @@ vespalib::string DocumentOperation::docArgsToString() const { } void -DocumentOperation::serialize(vespalib::nbostream &os) const +DocumentOperation::serialize(vespalib::nbostream &os) const { + serializeDocumentOperationOnly(os); +} + +void +DocumentOperation::serializeDocumentOperationOnly(vespalib::nbostream &os) const { os << _bucketId; os << _timestamp; @@ -74,8 +77,7 @@ DocumentOperation::serialize(vespalib::nbostream &os) const void -DocumentOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &) +DocumentOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &) { is >> _bucketId; is >> _timestamp; @@ -85,4 +87,8 @@ DocumentOperation::deserialize(vespalib::nbostream &is, is >> _prevTimestamp; } + DbDocumentId DocumentOperation::getDbDocumentId() const { + return _dbdId; + } + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h index bc961c580fc..9a823c553bd 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h @@ -22,124 +22,35 @@ protected: DocumentOperation(Type type); - DocumentOperation(Type type, - const document::BucketId &bucketId, + DocumentOperation(Type type, const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp); void assertValidBucketId(const document::DocumentId &docId) const; vespalib::string docArgsToString() const; public: - virtual - ~DocumentOperation() - { - } - - const - document::BucketId & - getBucketId() const - { - return _bucketId; - } - - storage::spi::Timestamp - getTimestamp() const - { - return _timestamp; - } - - search::DocumentIdT - getLid() const - { - return _dbdId.getLid(); - } - - search::DocumentIdT - getPrevLid() const - { - return _prevDbdId.getLid(); - } - - uint32_t - getSubDbId() const - { - return _dbdId.getSubDbId(); - } - - uint32_t - getPrevSubDbId() const - { - return _prevDbdId.getSubDbId(); - } - - bool - getValidDbdId() const - { - return _dbdId.valid(); - } - - bool - getValidDbdId(uint32_t subDbId) const - { - return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; - } - - bool - getValidPrevDbdId() const - { - return _prevDbdId.valid(); - } - - bool - getValidPrevDbdId(uint32_t subDbId) const - { - return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; - } - - bool - changedDbdId() const - { - return _dbdId != _prevDbdId; - } - bool - getPrevMarkedAsRemoved() const - { - return _prevMarkedAsRemoved; - } - - void - setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) - { - _prevMarkedAsRemoved = prevMarkedAsRemoved; - } - - DbDocumentId - getDbDocumentId() const - { - return _dbdId; - } - - DbDocumentId - getPrevDbDocumentId() const - { - return _prevDbdId; - } - - void - setDbDocumentId(DbDocumentId dbdId) - { - _dbdId = dbdId; - } - - void - setPrevDbDocumentId(DbDocumentId prevDbdId) - { - _prevDbdId = prevDbdId; - } - - search::DocumentIdT - getNewOrPrevLid(uint32_t subDbId) const - { + ~DocumentOperation() override {} + const document::BucketId &getBucketId() const { return _bucketId; } + storage::spi::Timestamp getTimestamp() const { return _timestamp; } + + search::DocumentIdT getLid() const { return _dbdId.getLid(); } + search::DocumentIdT getPrevLid() const { return _prevDbdId.getLid(); } + uint32_t getSubDbId() const { return _dbdId.getSubDbId(); } + uint32_t getPrevSubDbId() const { return _prevDbdId.getSubDbId(); } + bool getValidDbdId() const { return _dbdId.valid(); } + bool getValidDbdId(uint32_t subDbId) const { return _dbdId.valid() && _dbdId.getSubDbId() == subDbId; } + bool getValidPrevDbdId() const { return _prevDbdId.valid(); } + bool getValidPrevDbdId(uint32_t subDbId) const { return _prevDbdId.valid() && _prevDbdId.getSubDbId() == subDbId; } + bool changedDbdId() const { return _dbdId != _prevDbdId; } + bool getPrevMarkedAsRemoved() const { return _prevMarkedAsRemoved; } + void setPrevMarkedAsRemoved(bool prevMarkedAsRemoved) { _prevMarkedAsRemoved = prevMarkedAsRemoved; } + DbDocumentId getDbDocumentId() const; + DbDocumentId getPrevDbDocumentId() const { return _prevDbdId; } + + void setDbDocumentId(DbDocumentId dbdId) { _dbdId = dbdId; } + void setPrevDbDocumentId(DbDocumentId prevDbdId) { _prevDbdId = prevDbdId; } + + search::DocumentIdT getNewOrPrevLid(uint32_t subDbId) const { if (getValidDbdId() && getSubDbId() == subDbId) return getLid(); if (getValidPrevDbdId() && getPrevSubDbId() == subDbId) @@ -147,51 +58,34 @@ public: return 0; } - bool - getValidNewOrPrevDbdId() const - { + bool getValidNewOrPrevDbdId() const { return getValidDbdId() || getValidPrevDbdId(); } - bool - notMovingLidInSameSubDb() const - { + bool notMovingLidInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || getSubDbId() != getPrevSubDbId() || getLid() == getPrevLid(); } - bool - movingLidIfInSameSubDb() const - { + bool movingLidIfInSameSubDb() const { return !getValidDbdId() || !getValidPrevDbdId() || getSubDbId() != getPrevSubDbId() || getLid() != getPrevLid(); } - storage::spi::Timestamp - getPrevTimestamp() const - { - return _prevTimestamp; - } - - void - setPrevTimestamp(storage::spi::Timestamp prevTimestamp) - { - _prevTimestamp = prevTimestamp; - } - - virtual void - serialize(vespalib::nbostream &os) const override; + storage::spi::Timestamp getPrevTimestamp() const { return _prevTimestamp; } + void setPrevTimestamp(storage::spi::Timestamp prevTimestamp) { _prevTimestamp = prevTimestamp; } - virtual void - deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) override; + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; uint32_t getSerializedDocSize() const { return _serializedDocSize; } + + // Provided as a hook for tests. + void serializeDocumentOperationOnly(vespalib::nbostream &os) const; }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h index e44b7874c12..8a00c739126 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h @@ -52,8 +52,7 @@ public: void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; } SerialNum getSerialNum() const { return _serialNum; } virtual void serialize(vespalib::nbostream &os) const = 0; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) = 0; + virtual void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) = 0; virtual vespalib::string toString() const = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index beaf719dc5c..3468cc68ced 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -28,21 +28,15 @@ UpdateOperation::UpdateOperation(Type type) } -UpdateOperation::UpdateOperation(Type type, - const BucketId &bucketId, - const Timestamp ×tamp, - const DocumentUpdate::SP &upd) - : DocumentOperation(type, - bucketId, - timestamp), +UpdateOperation::UpdateOperation(Type type, const BucketId &bucketId, + const Timestamp ×tamp, const DocumentUpdate::SP &upd) + : DocumentOperation(type, bucketId, timestamp), _upd(upd) { } -UpdateOperation::UpdateOperation(const BucketId &bucketId, - const Timestamp ×tamp, - const DocumentUpdate::SP &upd) +UpdateOperation::UpdateOperation(const BucketId &bucketId, const Timestamp ×tamp, const DocumentUpdate::SP &upd) : UpdateOperation(FeedOperation::UPDATE, bucketId, timestamp, upd) { } @@ -50,20 +44,15 @@ UpdateOperation::UpdateOperation(const BucketId &bucketId, void UpdateOperation::serializeUpdate(vespalib::nbostream &os) const { - if (getType() == FeedOperation::UPDATE_42) { - _upd->serialize42(os); - } else { - _upd->serializeHEAD(os); - } + assert(getType() == UPDATE); + _upd->serializeHEAD(os); } void UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) { document::ByteBuffer buf(is.peek(), is.size()); - using Version = DocumentUpdate::SerializeVersion; - Version version = ((getType() == FeedOperation::UPDATE_42) ? Version::SERIALIZE_42 : Version::SERIALIZE_HEAD); - DocumentUpdate::SP update(std::make_shared<DocumentUpdate>(repo, buf, version)); + DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf); is.adjustReadPos(buf.getPos()); _upd = std::move(update); } @@ -78,8 +67,7 @@ UpdateOperation::serialize(vespalib::nbostream &os) const void -UpdateOperation::deserialize(vespalib::nbostream &is, - const DocumentTypeRepo &repo) +UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo) { DocumentOperation::deserialize(is, repo); try { @@ -108,12 +96,4 @@ vespalib::string UpdateOperation::toString() const { docArgsToString().c_str()); } -UpdateOperation -UpdateOperation::makeOldUpdate(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - const document::DocumentUpdate::SP &upd) -{ - return UpdateOperation(FeedOperation::UPDATE_42, bucketId, timestamp, upd); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 7886231af82..99dcbfbce6c 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -15,8 +15,7 @@ class UpdateOperation : public DocumentOperation private: using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>; DocumentUpdateSP _upd; - UpdateOperation(Type type, - const document::BucketId &bucketId, + UpdateOperation(Type type, const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); void serializeUpdate(vespalib::nbostream &os) const; @@ -27,15 +26,12 @@ public: UpdateOperation(const document::BucketId &bucketId, const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); - virtual ~UpdateOperation() {} + ~UpdateOperation() override {} const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; void deserializeUpdate(const document::DocumentTypeRepo &repo); - virtual vespalib::string toString() const override; - static UpdateOperation makeOldUpdate(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - const DocumentUpdateSP &upd); + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index 78084f29742..0dee7adfa49 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(searchcore_matching STATIC ranking_constants.cpp requestcontext.cpp result_processor.cpp + same_element_builder.cpp search_session.cpp session_manager_explorer.cpp sessionmanager.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 165fc67179a..268fe63ba4c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -2,6 +2,8 @@ #include "querynodes.h" #include "blueprintbuilder.h" +#include "termdatafromnode.h" +#include "same_element_builder.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> @@ -98,6 +100,15 @@ private: n.setDocumentFrequency(_result->getState().estimate().estHits, _context.getDocIdLimit()); } + void buildSameElement(ProtonSameElement &n) { + SameElementBuilder builder(_requestContext, _context); + for (size_t i = 0; i < n.getChildren().size(); ++i) { + search::query::Node &node = *n.getChildren()[i]; + builder.add_child(node); + } + _result = builder.build(); + } + template <typename NodeType> void buildTerm(NodeType &n) { FieldSpecList indexFields; @@ -131,7 +142,7 @@ protected: void visit(ProtonRank &n) override { buildIntermediate(new RankBlueprint(), n); } void visit(ProtonNear &n) override { buildIntermediate(new NearBlueprint(n.getDistance()), n); } void visit(ProtonONear &n) override { buildIntermediate(new ONearBlueprint(n.getDistance()), n); } - void visit(ProtonSameElement &n) override { buildIntermediate(nullptr /*new SameElementBlueprint())*/, n); } + void visit(ProtonSameElement &n) override { buildSameElement(n); } void visit(ProtonWeightedSetTerm &n) override { buildTerm(n); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h index eb45a735780..44cd6ffabfd 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.h @@ -20,4 +20,3 @@ struct BlueprintBuilder { }; } - diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp new file mode 100644 index 00000000000..d3a0ec4726f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp @@ -0,0 +1,96 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "same_element_builder.h" +#include "querynodes.h" +#include <vespa/searchlib/query/tree/customtypevisitor.h> +#include <vespa/searchlib/queryeval/leaf_blueprints.h> + +using search::queryeval::Blueprint; +using search::queryeval::EmptyBlueprint; +using search::queryeval::FieldSpecList; +using search::queryeval::IRequestContext; +using search::queryeval::SameElementBlueprint; +using search::queryeval::Searchable; + +namespace proton::matching { + +namespace { + +class SameElementBuilderVisitor : public search::query::CustomTypeVisitor<ProtonNodeTypes> +{ +private: + const IRequestContext &_requestContext; + ISearchContext &_context; + SameElementBlueprint &_result; + +public: + SameElementBuilderVisitor(const IRequestContext &requestContext, ISearchContext &context, SameElementBlueprint &result) + : _requestContext(requestContext), + _context(context), + _result(result) {} + + template <class TermNode> + void visitTerm(const TermNode &n) { + if (n.numFields() == 1) { + const ProtonTermData::FieldEntry &field = n.field(0); + assert(field.getFieldId() != search::fef::IllegalFieldId); + assert(field.getHandle() == search::fef::IllegalHandle); + FieldSpecList field_spec; + field_spec.add(_result.getNextChildField(field.field_name, field.getFieldId())); + Searchable &searchable = field.attribute_field ? _context.getAttributes() : _context.getIndexes(); + _result.addTerm(searchable.createBlueprint(_requestContext, field_spec, n)); + } + } + + void visit(ProtonAnd &) override {} + void visit(ProtonAndNot &) override {} + void visit(ProtonNear &) override {} + void visit(ProtonONear &) override {} + void visit(ProtonOr &) override {} + void visit(ProtonRank &) override {} + void visit(ProtonWeakAnd &) override {} + void visit(ProtonSameElement &) override {} + + void visit(ProtonWeightedSetTerm &) override {} + void visit(ProtonDotProduct &) override {} + void visit(ProtonWandTerm &) override {} + void visit(ProtonPhrase &) override {} + void visit(ProtonEquiv &) override {} + + void visit(ProtonNumberTerm &n) override { visitTerm(n); } + void visit(ProtonLocationTerm &n) override { visitTerm(n); } + void visit(ProtonPrefixTerm &n) override { visitTerm(n); } + void visit(ProtonRangeTerm &n) override { visitTerm(n); } + void visit(ProtonStringTerm &n) override { visitTerm(n); } + void visit(ProtonSubstringTerm &n) override { visitTerm(n); } + void visit(ProtonSuffixTerm &n) override { visitTerm(n); } + void visit(ProtonPredicateQuery &) override {} + void visit(ProtonRegExpTerm &n) override { visitTerm(n); } +}; + +} // namespace proton::matching::<unnamed> + +SameElementBuilder::SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context) + : _requestContext(requestContext), + _context(context), + _result(std::make_unique<SameElementBlueprint>()) +{ +} + +void +SameElementBuilder::add_child(search::query::Node &node) +{ + SameElementBuilderVisitor visitor(_requestContext, _context, *_result); + node.accept(visitor); +} + +Blueprint::UP +SameElementBuilder::build() +{ + if (!_result || _result->terms().empty()) { + return std::make_unique<EmptyBlueprint>(); + } + return std::move(_result); +} + +} // namespace proton::matching diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h new file mode 100644 index 00000000000..945bb9a97f6 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.h @@ -0,0 +1,24 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "isearchcontext.h" +#include <vespa/searchlib/query/tree/node.h> +#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/same_element_blueprint.h> + +namespace proton::matching { + +class SameElementBuilder +{ +private: + const search::queryeval::IRequestContext &_requestContext; + ISearchContext &_context; + std::unique_ptr<search::queryeval::SameElementBlueprint> _result; +public: + SameElementBuilder(const search::queryeval::IRequestContext &requestContext, ISearchContext &context); + void add_child(search::query::Node &node); + search::queryeval::Blueprint::UP build(); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp index 2b2849d025c..78733b14aaa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp @@ -13,23 +13,6 @@ using search::index::Schema; namespace proton { -FastAccessFeedView::UpdateScope -FastAccessFeedView::getUpdateScope(const DocumentUpdate &upd) -{ - UpdateScope updateScope; - for (const auto &update : upd.getUpdates()) { - const vespalib::string &fieldName = update.getField().getName(); - if (!fastPartialUpdateAttribute(fieldName)) { - updateScope._nonAttributeFields = true; - break; - } - } - if (!upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; - } - return updateScope; -} - /** * NOTE: For both put, update and remove we only need to pass the 'onWriteDone' * instance when we are going to commit as part of handling the operation. @@ -47,9 +30,18 @@ FastAccessFeedView::putAttributes(SerialNum serialNum, search::DocumentIdT lid, void FastAccessFeedView::updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) +{ + _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone, onUpdate); +} + +void +FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone) { - _attributeWriter->update(serialNum, upd, lid, immediateCommit, onWriteDone); + if (_attributeWriter->getHasCompoundAttribute()) { + _attributeWriter->update(serialNum, *doc.get(), lid, immediateCommit, onWriteDone); + } } void @@ -107,19 +99,4 @@ FastAccessFeedView::sync() _writeService.attributeFieldWriter().sync(); } -bool -FastAccessFeedView::fastPartialUpdateAttribute(const vespalib::string &fieldName) const { - search::AttributeVector *attribute = _attributeWriter->getWritableAttribute(fieldName); - if (attribute == nullptr) { - // Partial update to non-attribute field must update document - return false; - } - search::attribute::BasicType::Type attrType = attribute->getBasicType(); - // Partial update to tensor, predicate or reference attribute - // must update document - return ((attrType != search::attribute::BasicType::Type::PREDICATE) && - (attrType != search::attribute::BasicType::Type::TENSOR) && - (attrType != search::attribute::BasicType::Type::REFERENCE)); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h index e1b0cf83f64..3af97b4ecb9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.h @@ -39,14 +39,13 @@ private: const IAttributeWriter::SP _attributeWriter; DocIdLimit &_docIdLimit; - UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override; - void putAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, bool immediateCommit, OnPutDoneType onWriteDone) override; void updateAttributes(SerialNum serialNum, search::DocumentIdT lid, const document::DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate) override; + void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone) override; - void removeAttributes(SerialNum serialNum, search::DocumentIdT lid, bool immediateCommit, OnRemoveDoneType onWriteDone) override; @@ -73,8 +72,6 @@ public: void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; void sync() override; - - bool fastPartialUpdateAttribute(const vespalib::string &fieldName) const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 0c87d24899d..4cda07eee8b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -100,7 +100,7 @@ void SearchableFeedView::performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) { - Document::UP doc = std::move(futureDoc.get()); + const auto &doc = futureDoc.get(); if (doc) { performIndexPut(serialNum, lid, *doc, immediateCommit, onWriteDone); } @@ -118,29 +118,6 @@ SearchableFeedView::performIndexHeartBeat(SerialNum serialNum) _indexWriter->heartBeat(serialNum); } -SearchableFeedView::UpdateScope -SearchableFeedView::getUpdateScope(const DocumentUpdate &upd) -{ - UpdateScope updateScope; - const Schema &schema = *_schema; - for(size_t i(0), m(upd.getUpdates().size()); - !(updateScope._indexedFields && updateScope._nonAttributeFields) && - (i < m); i++) { - const document::FieldUpdate & fu(upd.getUpdates()[i]); - const vespalib::string &name = fu.getField().getName(); - if (schema.isIndexField(name)) { - updateScope._indexedFields = true; - } - if (!fastPartialUpdateAttribute(name)) { - updateScope._nonAttributeFields = true; - } - } - if (!upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; - } - return updateScope; -} - void SearchableFeedView::updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc futureDoc, bool immediateCommit, OnOperationDoneType onWriteDone) diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index 81d81a6cc27..ed3ba6740b1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -34,24 +34,19 @@ private: bool hasIndexedFields() const { return _hasIndexedFields; } - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, - bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document &doc, + bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc, - bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc, - bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &doc, + bool immediateCommit, OnOperationDoneType onWriteDone); + void performIndexPut(SerialNum serialNum, search::DocumentIdT lid, FutureDoc doc, + bool immediateCommit, OnOperationDoneType onWriteDone); - void - performIndexRemove(SerialNum serialNum, search::DocumentIdT lid, - bool immediateCommit, OnRemoveDoneType onWriteDone); + void performIndexRemove(SerialNum serialNum, search::DocumentIdT lid, + bool immediateCommit, OnRemoveDoneType onWriteDone); - void - performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove, - bool immediateCommit, OnWriteDoneType onWriteDone); + void performIndexRemove(SerialNum serialNum, const LidVector &lidsToRemove, + bool immediateCommit, OnWriteDoneType onWriteDone); void performIndexHeartBeat(SerialNum serialNum); @@ -60,23 +55,17 @@ private: void performSync(); void heartBeatIndexedFields(SerialNum serialNum) override; - virtual void - putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc, - bool immediateCommit, OnOperationDoneType onWriteDone) override; + void putIndexedFields(SerialNum serialNum, search::DocumentIdT lid, const document::Document::SP &newDoc, + bool immediateCommit, OnOperationDoneType onWriteDone) override; - UpdateScope getUpdateScope(const document::DocumentUpdate &upd) override; + void updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc, + bool immediateCommit, OnOperationDoneType onWriteDone) override; - virtual void - updateIndexedFields(SerialNum serialNum, search::DocumentIdT lid, FutureDoc newDoc, - bool immediateCommit, OnOperationDoneType onWriteDone) override; + void removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid, + bool immediateCommit, OnRemoveDoneType onWriteDone) override; - virtual void - removeIndexedFields(SerialNum serialNum, search::DocumentIdT lid, - bool immediateCommit, OnRemoveDoneType onWriteDone) override; - - virtual void - removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove, - bool immediateCommit, OnWriteDoneType onWriteDone) override; + void removeIndexedFields(SerialNum serialNum, const LidVector &lidsToRemove, + bool immediateCommit, OnWriteDoneType onWriteDone) override; void performIndexForceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone); void forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) override; @@ -85,7 +74,7 @@ public: SearchableFeedView(const StoreOnlyFeedView::Context &storeOnlyCtx, const PersistentParams ¶ms, const FastAccessFeedView::Context &fastUpdateCtx, Context ctx); - virtual ~SearchableFeedView(); + ~SearchableFeedView() override; const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } void sync() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index fdaf07dc466..a0f6ee98b71 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -17,6 +17,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> + LOG_SETUP(".proton.server.storeonlyfeedview"); using document::BucketId; @@ -300,20 +302,19 @@ StoreOnlyFeedView::heartBeatIndexedFields(SerialNum ) {} void StoreOnlyFeedView::heartBeatAttributes(SerialNum ) {} - -StoreOnlyFeedView::UpdateScope -StoreOnlyFeedView::getUpdateScope(const DocumentUpdate &upd) +void +StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate & upd, bool, + OnOperationDoneType, IFieldUpdateCallback & onUpdate) { - UpdateScope updateScope; - if (!upd.getUpdates().empty() || !upd.getFieldPathUpdates().empty()) { - updateScope._nonAttributeFields = true; + for (const auto & fieldUpdate : upd.getUpdates()) { + onUpdate.onUpdateField(fieldUpdate.getField().getName(), nullptr); } - return updateScope; } - void -StoreOnlyFeedView::updateAttributes(SerialNum, Lid, const DocumentUpdate &, bool, OnOperationDoneType) {} +StoreOnlyFeedView::updateAttributes(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType) +{ +} void StoreOnlyFeedView::updateIndexedFields(SerialNum, Lid, FutureDoc, bool, OnOperationDoneType) @@ -385,6 +386,34 @@ void StoreOnlyFeedView::heartBeatSummary(SerialNum serialNum) { })); } +StoreOnlyFeedView::UpdateScope::UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd) + : _schema(&schema), + _indexedFields(false), + _nonAttributeFields(!upd.getFieldPathUpdates().empty()) +{} + +namespace { + +bool isAttributeUpdateable(const search::AttributeVector *attribute) { + search::attribute::BasicType::Type attrType = attribute->getBasicType(); + // Partial update to tensor, predicate or reference attribute + // must update document + return ((attrType != search::attribute::BasicType::Type::PREDICATE) && + (attrType != search::attribute::BasicType::Type::TENSOR) && + (attrType != search::attribute::BasicType::Type::REFERENCE)); +} +} + +void +StoreOnlyFeedView::UpdateScope::onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) { + if (!_nonAttributeFields && (attr == nullptr || !isAttributeUpdateable(attr))) { + _nonAttributeFields = true; + } + if (!_indexedFields && _schema->isIndexField(fieldName)) { + _indexedFields = true; + } +} + void StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) { if ( ! updOp.getUpdate()) { @@ -417,15 +446,15 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) bool immediateCommit = _commitTimeTracker.needCommit(); auto onWriteDone = createUpdateDoneContext(std::move(token), updOp.getUpdate()); - updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone); + UpdateScope updateScope(*_schema, upd); + updateAttributes(serialNum, lid, upd, immediateCommit, onWriteDone, updateScope); - UpdateScope updateScope(getUpdateScope(upd)); if (updateScope.hasIndexOrNonAttributeFields()) { PromisedDoc promisedDoc; - FutureDoc futureDoc = promisedDoc.get_future(); + FutureDoc futureDoc = promisedDoc.get_future().share(); _pendingLidTracker.waitForConsumedLid(lid); if (updateScope._indexedFields) { - updateIndexedFields(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone); + updateIndexedFields(serialNum, lid, futureDoc, immediateCommit, onWriteDone); } PromisedStream promisedStream; FutureStream futureStream = promisedStream.get_future(); @@ -444,6 +473,7 @@ StoreOnlyFeedView::internalUpdate(FeedToken token, const UpdateOperation &updOp) std::move(promisedDoc), std::move(promisedStream)); }); #pragma GCC diagnostic pop + updateAttributes(serialNum, lid, std::move(futureDoc), immediateCommit, onWriteDone); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index b106b87c4fe..a11512590f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -9,6 +9,7 @@ #include "searchcontext.h" #include "pendinglidtracker.h" #include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/common/feeddebugger.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> @@ -33,6 +34,7 @@ class PutDoneContext; class RemoveDoneContext; class CommitTimeTracker; class IGidToLidChangeHandler; +class IFieldUpdateCallback; namespace documentmetastore { class ILidReuseDelayer; } @@ -59,7 +61,7 @@ public: using OnPutDoneType = const std::shared_ptr<PutDoneContext> &; using OnRemoveDoneType = const std::shared_ptr<RemoveDoneContext> &; using FeedTokenUP = std::unique_ptr<FeedToken>; - using FutureDoc = std::future<std::unique_ptr<Document>>; + using FutureDoc = std::shared_future<std::unique_ptr<Document>>; using PromisedDoc = std::promise<std::unique_ptr<Document>>; using FutureStream = std::future<vespalib::nbostream>; using PromisedStream = std::promise<vespalib::nbostream>; @@ -120,18 +122,19 @@ public: }; protected: - struct UpdateScope + class UpdateScope : public IFieldUpdateCallback { + private: + const search::index::Schema *_schema; + public: bool _indexedFields; bool _nonAttributeFields; - UpdateScope() - : _indexedFields(false), - _nonAttributeFields(false) - {} + UpdateScope(const search::index::Schema & schema, const DocumentUpdate & upd); bool hasIndexOrNonAttributeFields() const { return _indexedFields || _nonAttributeFields; } + void onUpdateField(vespalib::stringref fieldName, const search::AttributeVector * attr) override; }; private: @@ -200,9 +203,10 @@ private: virtual void putIndexedFields(SerialNum serialNum, Lid lid, const DocumentSP &newDoc, bool immediateCommit, OnOperationDoneType onWriteDone); - virtual UpdateScope getUpdateScope(const DocumentUpdate &upd); - virtual void updateAttributes(SerialNum serialNum, Lid lid, const DocumentUpdate &upd, + bool immediateCommit, OnOperationDoneType onWriteDone, IFieldUpdateCallback & onUpdate); + + virtual void updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc, bool immediateCommit, OnOperationDoneType onWriteDone); virtual void updateIndexedFields(SerialNum serialNum, Lid lid, FutureDoc doc, |