diff options
23 files changed, 402 insertions, 164 deletions
diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index 64e67672f0f..abe326425c3 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -133,7 +133,7 @@ struct UnitDR : DocumentRetrieverBaseForTest { } return DocumentMetaData(); } - document::Document::UP getDocument(DocumentIdT lid) const override { + document::Document::UP getFullDocument(DocumentIdT lid) const override { return Document::UP((lid == docid) ? document->clone() : nullptr); } @@ -145,7 +145,7 @@ struct UnitDR : DocumentRetrieverBaseForTest { } CachedSelect::SP parseSelect(const vespalib::string &selection) const override { - CachedSelect::SP res(new CachedSelect); + auto res = std::make_shared<CachedSelect>(); res->set(selection, repo); return res; } @@ -154,7 +154,7 @@ struct UnitDR : DocumentRetrieverBaseForTest { }; UnitDR::UnitDR() - : repo(), document(new Document(*DataType::DOCUMENT, DocumentId())), timestamp(0), + : repo(), document(std::make_unique<Document>(*DataType::DOCUMENT, DocumentId())), timestamp(0), bucket(), removed(false), docid(0), docIdLimit(std::numeric_limits<uint32_t>::max()) {} UnitDR::UnitDR(document::Document::UP d, Timestamp t, Bucket b, bool r) @@ -179,11 +179,11 @@ struct VisitRecordingUnitDR : UnitDR { { } - document::Document::UP getDocument(DocumentIdT lid) const override { + document::Document::UP getFullDocument(DocumentIdT lid) const override { if (lid == docid) { visited_lids.insert(lid); } - return UnitDR::getDocument(lid); + return UnitDR::getFullDocument(lid); } }; @@ -238,7 +238,7 @@ struct AttrUnitDR : public UnitDR } CachedSelect::SP parseSelect(const vespalib::string &selection) const override { - CachedSelect::SP res(new CachedSelect); + auto res = std::make_shared<CachedSelect>(); res->set(selection, "foo", Document(document->getType(), DocumentId()), repo, &_amgr, true); return res; } @@ -262,13 +262,13 @@ struct PairDR : DocumentRetrieverBaseForTest { DocumentMetaData ret = first->getDocumentMetaData(id); return (ret.valid()) ? ret : second->getDocumentMetaData(id); } - document::Document::UP getDocument(DocumentIdT lid) const override { - Document::UP ret = first->getDocument(lid); - return ret ? std::move(ret) : second->getDocument(lid); + document::Document::UP getFullDocument(DocumentIdT lid) const override { + Document::UP ret = first->getFullDocument(lid); + return ret ? std::move(ret) : second->getFullDocument(lid); } CachedSelect::SP parseSelect(const vespalib::string &selection) const override { - CachedSelect::SP res(new CachedSelect); + auto res = std::make_shared<CachedSelect>(); res->set(selection, getDocumentTypeRepo()); return res; } @@ -296,16 +296,26 @@ size_t getSize(const document::DocumentId &id) { return id.getSerializedSize() + getSize(); } -IDocumentRetriever::SP nil() { return IDocumentRetriever::SP(new UnitDR()); } +IDocumentRetriever::SP nil() { return std::make_unique<UnitDR>(); } -IDocumentRetriever::SP doc(const std::string &id, Timestamp t, Bucket b) { - Document::UP d(new Document(*DataType::DOCUMENT, DocumentId(id))); - return IDocumentRetriever::SP(new UnitDR(std::move(d), t, b, false)); +IDocumentRetriever::SP +doc(const DocumentId &id, Timestamp t, Bucket b) { + return std::make_shared<UnitDR>(std::make_unique<Document>(*DataType::DOCUMENT, id), t, b, false); } -IDocumentRetriever::SP rem(const std::string &id, Timestamp t, Bucket b) { - Document::UP d(new Document(*DataType::DOCUMENT, DocumentId(id))); - return IDocumentRetriever::SP(new UnitDR(std::move(d), t, b, true)); +IDocumentRetriever::SP +doc(const std::string &id, Timestamp t, Bucket b) { + return doc(DocumentId(id), t, b); +} + +IDocumentRetriever::SP +rem(const DocumentId &id, Timestamp t, Bucket b) { + return std::make_shared<UnitDR>(std::make_unique<Document>(*DataType::DOCUMENT, id), t, b, true); +} + +IDocumentRetriever::SP +rem(const std::string &id, Timestamp t, Bucket b) { + return rem(DocumentId(id), t, b); } IDocumentRetriever::SP cat(IDocumentRetriever::SP first, IDocumentRetriever::SP second) { @@ -337,15 +347,14 @@ const DocumentType &getAttrDocType() { } IDocumentRetriever::SP doc_with_fields(const std::string &id, Timestamp t, Bucket b) { - Document::UP d(new Document(getDocType(), DocumentId(id))); + auto d = std::make_unique<Document>(getDocType(), DocumentId(id)); d->set("header", "foo"); d->set("body", "bar"); - return IDocumentRetriever::SP(new UnitDR(getDocType(), std::move(d), t, b, false)); + return std::make_shared<UnitDR>(getDocType(), std::move(d), t, b, false); } IDocumentRetriever::SP doc_with_null_fields(const std::string &id, Timestamp t, Bucket b) { - Document::UP d(new Document(getAttrDocType(), DocumentId(id))); - return IDocumentRetriever::SP(new AttrUnitDR(std::move(d), t, b, false)); + return std::make_unique<AttrUnitDR>(std::make_unique<Document>(getAttrDocType(), DocumentId(id)), t, b, false); } IDocumentRetriever::SP doc_with_attr_fields(const vespalib::string &id, @@ -355,35 +364,32 @@ IDocumentRetriever::SP doc_with_attr_fields(const vespalib::string &id, const vespalib::string &ss, const vespalib::string &attr_ss) { - Document::UP d(new Document(getAttrDocType(), DocumentId(id))); + auto d = std::make_unique<Document>(getAttrDocType(), DocumentId(id)); d->set("header", "foo"); d->set("body", "bar"); d->set("aa", aa); d->set("ab", ab); d->set("dd", dd); d->set("ss", ss); - return IDocumentRetriever::SP(new AttrUnitDR(std::move(d), t, b, false, - attr_aa, attr_dd, attr_ss)); + return std::make_shared<AttrUnitDR>(std::move(d), t, b, false, attr_aa, attr_dd, attr_ss); } -auto doc_rec(VisitRecordingUnitDR::VisitedLIDs& visited_lids, - const std::string &id, Timestamp t, Bucket b) +auto doc_rec(VisitRecordingUnitDR::VisitedLIDs& visited_lids, const std::string &id, Timestamp t, Bucket b) { - Document::UP d(new Document(getDocType(), DocumentId(id))); - return std::make_shared<VisitRecordingUnitDR>( - visited_lids, std::move(d), t, b, false); + return std::make_shared<VisitRecordingUnitDR>(visited_lids, std::make_unique<Document>(getAttrDocType(), DocumentId(id)), t, b, false); } void checkDoc(const IDocumentRetriever &dr, const std::string &id, size_t timestamp, size_t bucket, bool removed) { - DocumentMetaData dmd = dr.getDocumentMetaData(DocumentId(id)); + DocumentId documentId(id); + DocumentMetaData dmd = dr.getDocumentMetaData(documentId); EXPECT_TRUE(dmd.valid()); EXPECT_EQUAL(timestamp, dmd.timestamp); EXPECT_EQUAL(bucket, dmd.bucketId.getId()); EXPECT_EQUAL(DocumentId(id).getGlobalId(), dmd.gid); EXPECT_EQUAL(removed, dmd.removed); - Document::UP doc = dr.getDocument(dmd.lid); + Document::UP doc = dr.getDocument(dmd.lid, documentId); ASSERT_TRUE(doc); EXPECT_TRUE(DocumentId(id) == doc->getId()); } @@ -415,15 +421,18 @@ void checkEntry(const IterateResult &res, size_t idx, const Document &doc, const } TEST("require that custom retrievers work as expected") { + DocumentId id1("id:ns:document::1"); + DocumentId id2("id:ns:document::2"); + DocumentId id3("id:ns:document::3"); IDocumentRetriever::SP dr = - cat(cat(doc("id:ns:document::1", Timestamp(2), bucket(5)), - rem("id:ns:document::2", Timestamp(3), bucket(5))), - cat(doc("id:ns:document::3", Timestamp(7), bucket(6)), + cat(cat(doc(id1, Timestamp(2), bucket(5)), + rem(id2, Timestamp(3), bucket(5))), + cat(doc(id3, Timestamp(7), bucket(6)), nil())); EXPECT_FALSE(dr->getDocumentMetaData(DocumentId("id:ns:document::bogus")).valid()); - EXPECT_FALSE(dr->getDocument(1)); - EXPECT_FALSE(dr->getDocument(2)); - EXPECT_TRUE(dr->getDocument(3)); + EXPECT_FALSE(dr->getDocument(1, id1)); + EXPECT_FALSE(dr->getDocument(2, id2)); + EXPECT_TRUE(dr->getDocument(3, id3)); TEST_DO(checkDoc(*dr, "id:ns:document::1", 2, 5, false)); TEST_DO(checkDoc(*dr, "id:ns:document::2", 3, 5, true)); TEST_DO(checkDoc(*dr, "id:ns:document::3", 7, 6, false)); diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 5fac9ef9c94..bdeb4a09685 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -104,7 +104,7 @@ struct MyDocumentRetriever : public DocumentRetrieverBaseForTest const document::DocumentTypeRepo &getDocumentTypeRepo() const override { return *_repo; } void getBucketMetaData(const storage::spi::Bucket &, DocumentMetaData::Vector &) const override {} DocumentMetaData getDocumentMetaData(const DocumentId &) const override { return DocumentMetaData(); } - Document::UP getDocument(DocumentIdT lid) const override { + Document::UP getFullDocument(DocumentIdT lid) const override { return Document::UP(_docs[lid]->clone()); } diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp index 7303d4ef6d0..e045ec31418 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp @@ -217,7 +217,7 @@ struct MyDocumentRetriever : public DocumentRetrieverBaseForTest { const document::DocumentTypeRepo& getDocumentTypeRepo() const override { return *repo; } void getBucketMetaData(const storage::spi::Bucket&, DocumentMetaData::Vector&) const override { abort(); } DocumentMetaData getDocumentMetaData(const DocumentId&) const override { abort(); } - Document::UP getDocument(DocumentIdT lid) const override { + Document::UP getFullDocument(DocumentIdT lid) const override { return store.read(lid, *repo); } CachedSelect::SP parseSelect(const vespalib::string&) const override { abort(); } diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 7e13f99d7b3..2540a991015 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -171,7 +171,7 @@ struct MyDocumentRetriever : public DocumentRetrieverBaseForTest } Document::UP - getDocument(DocumentIdT lid) const override + getFullDocument(DocumentIdT lid) const override { return _subDB.getDocument(lid); } diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 214ca186656..a31deca5d12 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -127,7 +127,7 @@ struct MyDocumentRetriever : DocumentRetrieverBaseForTest { } return DocumentMetaData(); } - document::Document::UP getDocument(search::DocumentIdT) const override { + document::Document::UP getFullDocument(search::DocumentIdT) const override { if (document != nullptr) { return Document::UP(document->clone()); } diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index 40224902b8f..a5279da52a2 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -17,6 +17,7 @@ #include <vespa/document/fieldvalue/structfieldvalue.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/eval/tensor/tensor.h> @@ -157,7 +158,7 @@ struct MyDocumentStore : proton::test::DummyDocumentStore { } const DocumentType *doc_type = r.getDocumentType(doc_type_name); auto doc = std::make_unique<Document>(*doc_type, doc_id); - ASSERT_TRUE(doc.get()); + ASSERT_TRUE(doc); doc->set(static_field, static_value); doc->set(dyn_field_i, static_value); doc->set(dyn_field_s, static_value_s); @@ -170,7 +171,7 @@ struct MyDocumentStore : proton::test::DummyDocumentStore { doc->setValue(dyn_field_tensor, tensorFieldValue); if (_set_position_struct_field) { FieldValue::UP fv = PositionDataType::getInstance().createFieldValue(); - StructFieldValue &pos = static_cast<StructFieldValue &>(*fv); + auto &pos = dynamic_cast<StructFieldValue &>(*fv); pos.set(PositionDataType::FIELD_X, 42); pos.set(PositionDataType::FIELD_Y, 21); doc->setValue(doc->getField(position_field), *fv); @@ -179,7 +180,7 @@ struct MyDocumentStore : proton::test::DummyDocumentStore { return doc; } - virtual uint64_t + uint64_t initFlush(uint64_t syncToken) override { return syncToken; @@ -278,11 +279,10 @@ struct Fixture { search::AttributeManager attr_manager; Schema schema; DocTypeName _dtName; - std::unique_ptr<DocumentRetriever> _retriever; + std::unique_ptr<IDocumentRetriever> _retriever; template <typename T> - T *addAttribute(const char *name, - Schema::DataType t, Schema::CollectionType ct) { + T *addAttribute(const char *name, Schema::DataType t, Schema::CollectionType ct) { AttributeVector::SP attrPtr = AttributeFactory::createAttribute(name, convertConfig(t, ct)); T *attr = dynamic_cast<T *>(attrPtr.get()); AttributeVector::DocId id; @@ -297,8 +297,7 @@ struct Fixture { } template <typename T, typename U> - void addAttribute(const char *name, U val, - Schema::DataType t, Schema::CollectionType ct) { + void addAttribute(const char *name, U val, Schema::DataType t, Schema::CollectionType ct) { T *attr = addAttribute<T>(name, t, ct); if (ct == schema::CollectionType::SINGLE) { attr->update(lid, val); @@ -309,7 +308,7 @@ struct Fixture { attr->commit(); } void addTensorAttribute(const char *name, const Tensor &val) { - TensorAttribute *attr = addAttribute<TensorAttribute>(name, schema::DataType::TENSOR, schema::CollectionType::SINGLE); + auto * attr = addAttribute<TensorAttribute>(name, schema::DataType::TENSOR, schema::CollectionType::SINGLE); attr->setTensor(lid, val); attr->commit(); } @@ -354,7 +353,7 @@ struct Fixture { addAttribute<StringAttribute>(dyn_field_nas, DataType::STRING, ct); addAttribute<IntegerAttribute>(zcurve_field, dynamic_zcurve_value, DataType::INT64, ct); addTensorAttribute(dyn_field_tensor.c_str(), *dynamic_tensor); - PredicateAttribute *attr = addAttribute<PredicateAttribute>(dyn_field_p, DataType::BOOLEANTREE, ct); + auto * attr = addAttribute<PredicateAttribute>(dyn_field_p, DataType::BOOLEANTREE, ct); attr->getIndex().indexEmptyDocument(lid); attr->commit(); ct = schema::CollectionType::ARRAY; @@ -371,7 +370,7 @@ struct Fixture { build(); } - void clearAttributes(std::vector<vespalib::string> names) { + void clearAttributes(const std::vector<vespalib::string> & names) const { for (const auto &name : names) { auto guard = *attr_manager.getAttribute(name); guard->clearDoc(lid); @@ -400,14 +399,14 @@ TEST_F("require that document retriever can retrieve bucket meta data", Fixture) TEST_F("require that document retriever can retrieve document", Fixture) { DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); EXPECT_EQUAL(doc_id, doc->getId()); } template <typename T> bool checkFieldValue(FieldValue::UP field_value, typename T::value_type v) { - ASSERT_TRUE(field_value.get()); + ASSERT_TRUE(field_value); T *t_value = dynamic_cast<T *>(field_value.get()); ASSERT_TRUE(t_value); return EXPECT_EQUAL(v, t_value->getValue()); @@ -415,8 +414,8 @@ bool checkFieldValue(FieldValue::UP field_value, typename T::value_type v) { template <typename T> void checkArray(FieldValue::UP array, typename T::value_type v) { - ASSERT_TRUE(array.get()); - ArrayFieldValue *array_val = dynamic_cast<ArrayFieldValue *>(array.get()); + ASSERT_TRUE(array); + auto *array_val = dynamic_cast<ArrayFieldValue *>(array.get()); ASSERT_TRUE(array_val); ASSERT_EQUAL(2u, array_val->size()); T *t_value = dynamic_cast<T *>(&(*array_val)[0]); @@ -428,9 +427,8 @@ void checkArray(FieldValue::UP array, typename T::value_type v) { template <typename T> void checkWset(FieldValue::UP wset, T v) { - ASSERT_TRUE(wset.get()); - WeightedSetFieldValue *wset_val = - dynamic_cast<WeightedSetFieldValue *>(wset.get()); + ASSERT_TRUE(wset); + auto *wset_val = dynamic_cast<WeightedSetFieldValue *>(wset.get()); ASSERT_TRUE(wset_val); ASSERT_EQUAL(2u, wset_val->size()); EXPECT_EQUAL(dyn_weight, wset_val->get(v)); @@ -439,12 +437,12 @@ void checkWset(FieldValue::UP wset, T v) { TEST_F("require that attributes are patched into stored document", Fixture) { DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); FieldValue::UP value = doc->getValue(static_field); - ASSERT_TRUE(value.get()); - IntFieldValue *int_value = dynamic_cast<IntFieldValue *>(value.get()); + ASSERT_TRUE(value); + auto *int_value = dynamic_cast<IntFieldValue *>(value.get()); ASSERT_TRUE(int_value); EXPECT_EQUAL(static_value, int_value->getValue()); @@ -467,21 +465,31 @@ TEST_F("require that attributes are patched into stored document", Fixture) { EXPECT_FALSE(doc->getValue(dyn_wset_field_n)); } +TEST_F("require that we can look up NONE and DOCIDONLY field sets", Fixture) { + DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); + Document::UP doc = f._retriever->getPartialDocument(meta_data.lid, doc_id, document::NoFields()); + ASSERT_TRUE(doc); + EXPECT_TRUE(doc->getFields().empty()); + doc = f._retriever->getPartialDocument(meta_data.lid, doc_id, document::DocIdOnly()); + ASSERT_TRUE(doc); + EXPECT_TRUE(doc->getFields().empty()); +} + TEST_F("require that attributes are patched into stored document unless also index field", Fixture) { f.addIndexField(Schema::IndexField(dyn_field_s, DataType::STRING)).build(); DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); checkFieldValue<StringFieldValue>(doc->getValue(dyn_field_s), static_value_s); } void verify_position_field_has_expected_values(Fixture& f) { DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); FieldValue::UP value = doc->getValue(position_field); - ASSERT_TRUE(value.get()); + ASSERT_TRUE(value); const auto *position = dynamic_cast<StructFieldValue *>(value.get()); ASSERT_TRUE(position); FieldValue::UP x = position->getValue(PositionDataType::FIELD_X); @@ -503,10 +511,10 @@ TEST_F("zcurve attribute is authoritative for single value position field existe TEST_F("require that array position field value is generated from zcurve array attribute", Fixture) { DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); FieldValue::UP value = doc->getValue(position_array_field); - ASSERT_TRUE(value.get()); + ASSERT_TRUE(value); const auto* array_value = dynamic_cast<const document::ArrayFieldValue*>(value.get()); ASSERT_TRUE(array_value != nullptr); ASSERT_EQUAL(array_value->getNestedType(), document::PositionDataType::getInstance()); @@ -528,33 +536,32 @@ TEST_F("require that array position field value is generated from zcurve array a } TEST_F("require that non-existing lid returns null pointer", Fixture) { - Document::UP doc = f._retriever->getDocument(0); - ASSERT_FALSE(doc.get()); + Document::UP doc = f._retriever->getDocument(0, DocumentId("id:ns:document::1")); + ASSERT_FALSE(doc); } TEST_F("require that predicate attributes can be retrieved", Fixture) { DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); FieldValue::UP value = doc->getValue(dyn_field_p); - ASSERT_TRUE(value.get()); - PredicateFieldValue *predicate_value = - dynamic_cast<PredicateFieldValue *>(value.get()); + ASSERT_TRUE(value); + auto *predicate_value = dynamic_cast<PredicateFieldValue *>(value.get()); ASSERT_TRUE(predicate_value); } TEST_F("require that zero values in multivalue attribute removes fields", Fixture) { auto meta_data = f._retriever->getDocumentMetaData(doc_id); - auto doc = f._retriever->getDocument(meta_data.lid); + auto doc = f._retriever->getDocument(meta_data.lid, doc_id); ASSERT_TRUE(doc); const Document *docPtr = doc.get(); ASSERT_TRUE(doc->hasValue(dyn_arr_field_i)); ASSERT_TRUE(doc->hasValue(dyn_wset_field_i)); f.doc_store._testDoc = std::move(doc); f.clearAttributes({ dyn_arr_field_i, dyn_wset_field_i }); - doc = f._retriever->getDocument(meta_data.lid); + doc = f._retriever->getDocument(meta_data.lid, doc_id); EXPECT_EQUAL(docPtr, doc.get()); ASSERT_FALSE(doc->hasValue(dyn_arr_field_i)); ASSERT_FALSE(doc->hasValue(dyn_wset_field_i)); @@ -562,15 +569,58 @@ TEST_F("require that zero values in multivalue attribute removes fields", Fixtur TEST_F("require that tensor attribute can be retrieved", Fixture) { DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id); - Document::UP doc = f._retriever->getDocument(meta_data.lid); - ASSERT_TRUE(doc.get()); + Document::UP doc = f._retriever->getDocument(meta_data.lid, doc_id); + ASSERT_TRUE(doc); FieldValue::UP value = doc->getValue(dyn_field_tensor); ASSERT_TRUE(value); - TensorFieldValue *tensor_value = dynamic_cast<TensorFieldValue *>(value.get()); + auto * tensor_value = dynamic_cast<TensorFieldValue *>(value.get()); ASSERT_TRUE(tensor_value->getAsTensorPtr()->equals(*dynamic_tensor)); } +struct Lookup : public IFieldInfo +{ + Lookup() : _count(0) {} + bool isFieldAttribute(const document::Field & field) const override { + _count++; + return (field.getName()[0] == 'a'); + } + mutable unsigned _count; +}; + +TEST("require that fieldset can figure out their attributeness and rember it") { + Lookup lookup; + FieldSetAttributeDB fsDB(lookup); + document::Field attr1("attr1", 1, *document::DataType::LONG, true); + document::Field attr2("attr2", 2, *document::DataType::LONG, true); + document::Field not_attr1("not_attr1", 3, *document::DataType::LONG, true); + document::Field::Set allAttr; + allAttr.insert(&attr1); + EXPECT_TRUE(fsDB.areAllFieldsAttributes(13, allAttr)); + EXPECT_EQUAL(1u, lookup._count); + EXPECT_TRUE(fsDB.areAllFieldsAttributes(13, allAttr)); + EXPECT_EQUAL(1u, lookup._count); + + allAttr.insert(&attr2); + EXPECT_TRUE(fsDB.areAllFieldsAttributes(17, allAttr)); + EXPECT_EQUAL(3u, lookup._count); + EXPECT_TRUE(fsDB.areAllFieldsAttributes(17, allAttr)); + EXPECT_EQUAL(3u, lookup._count); + + document::Field::Set notAllAttr; + notAllAttr.insert(¬_attr1); + EXPECT_FALSE(fsDB.areAllFieldsAttributes(33, notAllAttr)); + EXPECT_EQUAL(4u, lookup._count); + EXPECT_FALSE(fsDB.areAllFieldsAttributes(33, notAllAttr)); + EXPECT_EQUAL(4u, lookup._count); + + notAllAttr.insert(&attr1); + EXPECT_FALSE(fsDB.areAllFieldsAttributes(39, notAllAttr)); + EXPECT_EQUAL(6u, lookup._count); + EXPECT_FALSE(fsDB.areAllFieldsAttributes(39, notAllAttr)); + EXPECT_EQUAL(6u, lookup._count); +} + } // namespace TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp index e0243934fd6..0b49fecc79b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_populator.cpp @@ -47,7 +47,7 @@ DocumentFieldPopulator::~DocumentFieldPopulator() void DocumentFieldPopulator::handleExisting(uint32_t lid, const std::shared_ptr<Document> &doc) { - DocumentFieldRetriever::populate(lid, *doc, doc->getField(_fieldName), *_attr, false); + DocumentFieldRetriever::populate(lid, *doc, doc->getField(_fieldName), *_attr); ++_documentsPopulated; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp index d1cb5cca3df..358556abce7 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.cpp @@ -120,8 +120,7 @@ void DocumentFieldRetriever::populate(DocumentIdT lid, Document &doc, const document::Field & field, - const IAttributeVector &attr, - bool isIndexField) + const IAttributeVector &attr) { switch (attr.getBasicType()) { case BasicType::BOOL: @@ -138,12 +137,6 @@ DocumentFieldRetriever::populate(DocumentIdT lid, setValue<double>(lid, doc, field, attr); break; case BasicType::STRING: - // If it is a stringfield we also need to check if - // it is an index field. In that case we shall - // keep the original in order to preserve annotations. - if (isIndexField) { - break; - } setValue<const char *>(lid, doc, field, attr); break; case BasicType::PREDICATE: diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h index b1973c62527..e80bd2b9d99 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_retriever.h @@ -16,8 +16,7 @@ struct DocumentFieldRetriever static void populate(search::DocumentIdT lid, document::Document &doc, const document::Field &field, - const search::attribute::IAttributeVector &attr, - bool isIndexField); + const search::attribute::IAttributeVector &attr); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.cpp index f3c8df31aa8..bf0faab1cff 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "commit_and_wait_document_retriever.h" +#include <vespa/document/fieldvalue/document.h> namespace proton { @@ -11,4 +12,57 @@ CommitAndWaitDocumentRetriever::CommitAndWaitDocumentRetriever(IDocumentRetrieve CommitAndWaitDocumentRetriever::~CommitAndWaitDocumentRetriever() = default; +const document::DocumentTypeRepo & +CommitAndWaitDocumentRetriever::getDocumentTypeRepo() const { + return _retriever->getDocumentTypeRepo(); +} + +void +CommitAndWaitDocumentRetriever::getBucketMetaData(const Bucket &bucket, search::DocumentMetaData::Vector &result) const { + return _retriever->getBucketMetaData(bucket, result); +} + +search::DocumentMetaData +CommitAndWaitDocumentRetriever::getDocumentMetaData(const document::DocumentId &id) const { + return _retriever->getDocumentMetaData(id); +} + +document::Document::UP +CommitAndWaitDocumentRetriever::getFullDocument(search::DocumentIdT lid) const { + // Ensure that attribute vectors are committed + _commit.commitAndWait(); + return _retriever->getFullDocument(lid); +} + +document::Document::UP +CommitAndWaitDocumentRetriever::getPartialDocument(search::DocumentIdT lid, const document::DocumentId & docId, + const document::FieldSet & fieldSet) const +{ + _commit.commitAndWait(); + return _retriever->getPartialDocument(lid, docId, fieldSet); +} + +void +CommitAndWaitDocumentRetriever::visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, + ReadConsistency readConsistency) const +{ + _commit.commitAndWait(); + _retriever->visitDocuments(lids, visitor, readConsistency); +} + +CachedSelect::SP +CommitAndWaitDocumentRetriever::parseSelect(const vespalib::string &selection) const { + return _retriever->parseSelect(selection); +} + +IDocumentRetriever::ReadGuard +CommitAndWaitDocumentRetriever::getReadGuard() const { + return _retriever->getReadGuard(); +} + +uint32_t +CommitAndWaitDocumentRetriever::getDocIdLimit() const { + return _retriever->getDocIdLimit(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h index 55e5ecdcdcc..c03330164c0 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h @@ -4,7 +4,6 @@ #include "i_document_retriever.h" #include <vespa/searchcore/proton/common/icommitable.h> -#include <vespa/document/fieldvalue/document.h> namespace proton { @@ -22,38 +21,15 @@ public: CommitAndWaitDocumentRetriever(IDocumentRetriever::SP retriever, ICommitable &commit); ~CommitAndWaitDocumentRetriever() override; - const document::DocumentTypeRepo &getDocumentTypeRepo() const override { - return _retriever->getDocumentTypeRepo(); - } - - void getBucketMetaData(const Bucket &bucket, search::DocumentMetaData::Vector &result) const override { - return _retriever->getBucketMetaData(bucket, result); - } - - search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override { - return _retriever->getDocumentMetaData(id); - } - document::Document::UP getDocument(search::DocumentIdT lid) const override { - // Ensure that attribute vectors are committed - _commit.commitAndWait(); - return _retriever->getDocument(lid); - } - void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, - ReadConsistency readConsistency) const override - { - _commit.commitAndWait(); - _retriever->visitDocuments(lids, visitor, readConsistency); - } - - CachedSelect::SP parseSelect(const vespalib::string &selection) const override { - return _retriever->parseSelect(selection); - } - ReadGuard getReadGuard() const override { - return _retriever->getReadGuard(); - } - uint32_t getDocIdLimit() const override { - return _retriever->getDocIdLimit(); - } + const document::DocumentTypeRepo &getDocumentTypeRepo() const override; + void getBucketMetaData(const Bucket &bucket, search::DocumentMetaData::Vector &result) const override; + search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override; + DocumentUP getFullDocument(search::DocumentIdT lid) const override; + DocumentUP getPartialDocument(search::DocumentIdT lid, const document::DocumentId & docId, const document::FieldSet & fieldSet) const override; + void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const override; + CachedSelect::SP parseSelect(const vespalib::string &selection) const override; + ReadGuard getReadGuard() const override; + uint32_t getDocIdLimit() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp index 579bab5ab8b..952d1e8a693 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.cpp @@ -3,13 +3,29 @@ #include "i_document_retriever.h" #include <vespa/persistence/spi/read_consistency.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldset/fieldsets.h> namespace proton { -void DocumentRetrieverBaseForTest::visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const { +document::Document::UP +IDocumentRetriever::getDocument(search::DocumentIdT lid, const document::DocumentId & docId) const { + return getPartialDocument(lid, docId, document::AllFields()); +} + +document::Document::UP +IDocumentRetriever::getPartialDocument(search::DocumentIdT lid, const document::DocumentId &, const document::FieldSet & fieldSet) const { + auto doc = getFullDocument(lid); + if (doc) { + document::FieldSet::stripFields(*doc, fieldSet); + } + return doc; +} + +void +DocumentRetrieverBaseForTest::visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const { (void) readConsistency; for (uint32_t lid : lids) { - visitor.visit(lid, getDocument(lid)); + visitor.visit(lid, getFullDocument(lid)); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h index 1ec03426d20..2d20ed7bc89 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h @@ -10,7 +10,7 @@ #include <vespa/searchcore/proton/common/cachedselect.h> #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store_context.h> -namespace document { class Document; } +namespace document { class FieldSet; } namespace proton { @@ -32,10 +32,17 @@ public: virtual ~IDocumentRetriever() = default; - virtual const document::DocumentTypeRepo &getDocumentTypeRepo() const = 0; + virtual const document::DocumentTypeRepo & getDocumentTypeRepo() const = 0; virtual void getBucketMetaData(const storage::spi::Bucket &bucket, search::DocumentMetaData::Vector &result) const = 0; virtual search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const = 0; - virtual DocumentUP getDocument(search::DocumentIdT lid) const = 0; + /** + * Extracts the full document based on the LID + */ + virtual DocumentUP getFullDocument(search::DocumentIdT lid) const = 0; + /** + * Fetches the necessary set of fields, allowing for more optimal fetch when combining only from attributes. + */ + virtual DocumentUP getPartialDocument(search::DocumentIdT lid, const document::DocumentId & docId, const document::FieldSet & fieldSet) const; virtual ReadGuard getReadGuard() const = 0; virtual uint32_t getDocIdLimit() const = 0; /** @@ -47,6 +54,9 @@ public: virtual void visitDocuments(const LidVector &lids, search::IDocumentVisitor &visitor, ReadConsistency readConsistency) const = 0; virtual CachedSelect::SP parseSelect(const vespalib::string &selection) const = 0; + + // Convenience to get all fields + DocumentUP getDocument(search::DocumentIdT lid, const document::DocumentId & docId) const; }; class DocumentRetrieverBaseForTest : public IDocumentRetriever { diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index da1b90aa167..b5166276e4a 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -439,11 +439,10 @@ PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const if (document::FieldSet::Type::NONE == fields.getType()) { return GetResult::make_for_metadata_only(meta.timestamp); } - document::Document::UP doc = retriever.getDocument(meta.lid); + document::Document::UP doc = retriever.getPartialDocument(meta.lid, did, fields); if (!doc || doc->getId().getGlobalId() != meta.gid) { return GetResult(); } - document::FieldSet::stripFields(*doc, fields); return GetResult(std::move(doc), meta.timestamp); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp index d6d8ba03a67..3d5c51c4346 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp @@ -24,7 +24,7 @@ DocumentBucketMover::moveDocument(DocumentIdT lid, const document::GlobalId &gid, Timestamp timestamp) { - Document::SP doc(_source->retriever()->getDocument(lid).release()); + Document::SP doc(_source->retriever()->getFullDocument(lid).release()); if (!doc || doc->getId().getGlobalId() != gid) return; // Failed to retrieve document, removed or changed identity // TODO(geirst): what if doc is NULL? diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index 7466bcd99d5..4484338ddfd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -6,6 +6,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/fieldset/fieldsets.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcore/proton/attribute/document_field_retriever.h> #include <vespa/vespalib/geo/zcurve.h> @@ -28,6 +29,7 @@ using search::IDocumentStore; using search::index::Schema; using storage::spi::Timestamp; using vespalib::geo::ZCurve; +using search::attribute::BasicType; namespace proton { @@ -43,6 +45,31 @@ bool is_array_of_position_type(const document::DataType& field_type) noexcept { } +FieldSetAttributeDB::FieldSetAttributeDB(const IFieldInfo & fieldInfo) + : _fieldInfo(fieldInfo), + _isFieldSetAttributeOnly(), + _lock() +{} + +FieldSetAttributeDB::~FieldSetAttributeDB() = default; + +bool +FieldSetAttributeDB::areAllFieldsAttributes(uint64_t key, const document::Field::Set & set) const { + std::lock_guard guard(_lock); + auto found = _isFieldSetAttributeOnly.find(key); + bool isAttributeOnly = true; + if (found == _isFieldSetAttributeOnly.end()) { + for (const document::Field *field : set) { + isAttributeOnly = _fieldInfo.isFieldAttribute(*field); + if (!isAttributeOnly) break; + } + _isFieldSetAttributeOnly[key] = isAttributeOnly; + } else { + isAttributeOnly = found->second; + } + return isAttributeOnly; +} + DocumentRetriever ::DocumentRetriever(const DocTypeName &docTypeName, const DocumentTypeRepo &repo, @@ -55,7 +82,9 @@ DocumentRetriever _attr_manager(attr_manager), _doc_store(doc_store), _possiblePositionFields(), - _attributeFields() + _attributeFields(), + _areAllFieldsAttributes(true), + _fieldSetAttributeInfo(*this) { const DocumentType * documentType = repo.getDocumentType(docTypeName.getName()); document::Field::Set fields = documentType->getFieldSet(); @@ -69,22 +98,56 @@ DocumentRetriever if (attr && attr->valid()) { LOG(debug, "Field '%s' is a registered attribute field", zcurve_name.c_str()); _possiblePositionFields.emplace_back(field, zcurve_name); + } else { + _areAllFieldsAttributes = false; } } else { const vespalib::string &name = field->getName(); AttributeGuard::UP attr = attr_manager.getAttribute(name); - if (attr && attr->valid()) { - _attributeFields.emplace_back(field); + if (attr && attr->valid() + && !_schema.isIndexField(field->getName()) + && ((*attr)->getBasicType() != BasicType::PREDICATE) + && ((*attr)->getBasicType() != BasicType::REFERENCE)) + { + _attributeFields.insert(field); + } else { + _areAllFieldsAttributes = false; } } } } +bool +DocumentRetriever::needFetchFromDocStore(const document::FieldSet & fieldSet) const { + switch (fieldSet.getType()) { + case document::FieldSet::Type::NONE: + case document::FieldSet::Type::DOCID: + return false; + case document::FieldSet::Type::ALL: + return ! _areAllFieldsAttributes; + case document::FieldSet::Type::FIELD: { + const auto & field = static_cast<const document::Field &>(fieldSet); + return ! isFieldAttribute(field); + } + case document::FieldSet::Type::SET: { + const auto &set = static_cast<const document::FieldCollection &>(fieldSet); + return ! _fieldSetAttributeInfo.areAllFieldsAttributes(set.hash(), set.getFields()); + } + } + abort(); +} + +bool +DocumentRetriever::isFieldAttribute(const document::Field & field) const { + return _attributeFields.find(&field) != _attributeFields.end(); +} + DocumentRetriever::~DocumentRetriever() = default; namespace { -FieldValue::UP positionFromZcurve(int64_t zcurve) { +std::unique_ptr<document::FieldValue> +positionFromZcurve(int64_t zcurve) { int32_t x, y; ZCurve::decode(zcurve, &x, &y); @@ -111,7 +174,8 @@ zcurve_array_attribute_to_field_value(const document::Field& field, return new_fv; } -void fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::PositionFields & possiblePositionFields, const IAttributeManager & attr_manager) +void +fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::PositionFields & possiblePositionFields, const IAttributeManager & attr_manager) { for (const auto & it : possiblePositionFields) { auto attr_guard = attr_manager.getAttribute(it.second); @@ -155,7 +219,7 @@ private: } // namespace Document::UP -DocumentRetriever::getDocument(DocumentIdT lid) const +DocumentRetriever::getFullDocument(DocumentIdT lid) const { Document::UP doc = _doc_store.read(lid, getDocumentTypeRepo()); if (doc) { @@ -164,18 +228,60 @@ DocumentRetriever::getDocument(DocumentIdT lid) const return doc; } -void DocumentRetriever::visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const +Document::UP +DocumentRetriever::getPartialDocument(search::DocumentIdT lid, const document::DocumentId & docId, const document::FieldSet & fieldSet) const { + Document::UP doc; + if (needFetchFromDocStore(fieldSet)) { + doc = _doc_store.read(lid, getDocumentTypeRepo()); + if (doc) { + populate(lid, *doc); + } + document::FieldSet::stripFields(*doc, fieldSet); + } else { + doc = std::make_unique<Document>(getDocumentType(), docId); + switch (fieldSet.getType()) { + case document::FieldSet::Type::ALL: + populate(lid, *doc); + break; + case document::FieldSet::Type::FIELD: { + const auto & field = static_cast<const document::Field &>(fieldSet); + document::Field::Set attributes; + attributes.insert(&field); + populate(lid, *doc, attributes); + break; + } + case document::FieldSet::Type::SET: { + const auto &set = static_cast<const document::FieldCollection &>(fieldSet); + populate(lid, *doc, set.getFields()); + break; + } + case document::FieldSet::Type::NONE: + case document::FieldSet::Type::DOCID: + break; + } + } + return doc; +} + +void +DocumentRetriever::visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const { PopulateVisitor populater(*this, visitor); _doc_store.visit(lids, getDocumentTypeRepo(), populater); } -void DocumentRetriever::populate(DocumentIdT lid, Document & doc) const +void +DocumentRetriever::populate(DocumentIdT lid, Document & doc) const { + populate(lid, doc, _attributeFields); +} + +void +DocumentRetriever::populate(DocumentIdT lid, Document & doc, const document::Field::Set & attributeFields) const { - for (const document::Field * field : _attributeFields) { + for (const document::Field * field : attributeFields) { AttributeGuard::UP attr = _attr_manager.getAttribute(field->getName()); if (lid < (*attr)->getCommittedDocIdLimit()) { - DocumentFieldRetriever::populate(lid, doc, *field, **attr, _schema.isIndexField(field->getName())); + DocumentFieldRetriever::populate(lid, doc, *field, **attr); } else { doc.remove(*field); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.h b/searchcore/src/vespa/searchcore/proton/server/documentretriever.h index 71603124b7b..bab8898c8ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.h @@ -3,20 +3,38 @@ #pragma once #include "documentretrieverbase.h" +#include <vespa/vespalib/stllike/hash_map.h> namespace search { -struct IDocumentMetaStore; -class IAttributeManager; -class IDocumentStore; -namespace index { class Schema; } -} // namespace search + struct IDocumentMetaStore; + class IAttributeManager; + class IDocumentStore; +} +namespace search::index { class Schema; } namespace proton { -class DocumentRetriever : public DocumentRetrieverBase { +struct IFieldInfo { + virtual ~IFieldInfo() = default; + virtual bool isFieldAttribute(const document::Field & field) const = 0; +}; + +class FieldSetAttributeDB { +public: + FieldSetAttributeDB(const IFieldInfo & fieldInfo); + ~FieldSetAttributeDB(); + bool areAllFieldsAttributes(uint64_t key, const document::Field::Set & set) const; +private: + using FieldSetAttributeMap = vespalib::hash_map<uint64_t, bool>; + const IFieldInfo & _fieldInfo; + mutable FieldSetAttributeMap _isFieldSetAttributeOnly; + mutable std::mutex _lock; +}; + +class DocumentRetriever : public DocumentRetrieverBase, + public IFieldInfo { public: typedef std::vector<std::pair<const document::Field *, vespalib::string>> PositionFields; - using AttributeFields = std::vector<const document::Field *>; DocumentRetriever(const DocTypeName &docTypeName, const document::DocumentTypeRepo &repo, const search::index::Schema &schema, @@ -25,15 +43,22 @@ public: const search::IDocumentStore &doc_store); ~DocumentRetriever() override; - document::Document::UP getDocument(search::DocumentIdT lid) const override; + document::Document::UP getFullDocument(search::DocumentIdT lid) const override; void visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const override; + DocumentUP getPartialDocument(search::DocumentIdT lid, const document::DocumentId &, const document::FieldSet &) const override; void populate(search::DocumentIdT lid, document::Document & doc) const; + bool needFetchFromDocStore(const document::FieldSet &) const; private: + void populate(search::DocumentIdT lid, document::Document & doc, const document::Field::Set & attributeFields) const; + + bool isFieldAttribute(const document::Field & field) const override; const search::index::Schema &_schema; const search::IAttributeManager &_attr_manager; const search::IDocumentStore &_doc_store; PositionFields _possiblePositionFields; - AttributeFields _attributeFields; + document::Field::Set _attributeFields; + bool _areAllFieldsAttributes; + FieldSetAttributeDB _fieldSetAttributeInfo; const search::IAttributeManager * getAttrMgr() const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp index 183e9eb0c0a..1476a699adb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.cpp @@ -87,5 +87,4 @@ DocumentRetrieverBase::parseSelect(const vespalib::string &selection) const return nselect; } - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h index 53341e0d335..351a0c9142b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentretrieverbase.h @@ -26,16 +26,18 @@ class DocumentRetrieverBase : public IDocumentRetriever protected: virtual const search::IAttributeManager * getAttrMgr() const; + const document::DocumentType & getDocumentType() const { + return _emptyDoc->getType(); + } public: DocumentRetrieverBase(const DocTypeName &docTypeName, const document::DocumentTypeRepo &repo, const IDocumentMetaStoreContext &meta_store, bool hasFields); - ~DocumentRetrieverBase(); + ~DocumentRetrieverBase() override; const document::DocumentTypeRepo &getDocumentTypeRepo() const override; - void getBucketMetaData(const storage::spi::Bucket &bucket, - search::DocumentMetaData::Vector &result) const override; + void getBucketMetaData(const storage::spi::Bucket &bucket, search::DocumentMetaData::Vector &result) const override; search::DocumentMetaData getDocumentMetaData(const document::DocumentId &id) const override; CachedSelect::SP parseSelect(const vespalib::string &selection) const override; ReadGuard getReadGuard() const override { return _meta_store.getReadGuard(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp index 98de2902f46..8aeaf7fd9e4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp @@ -46,7 +46,7 @@ MoveOperation::UP LidSpaceCompactionHandler::createMoveOperation(const search::DocumentMetaData &document, uint32_t moveToLid) const { const uint32_t moveFromLid = document.lid; - auto doc = _subDb.retriever()->getDocument(moveFromLid); + auto doc = _subDb.retriever()->getFullDocument(moveFromLid); auto op = std::make_unique<MoveOperation>(document.bucketId, document.timestamp, Document::SP(doc.release()), DbDocumentId(_subDb.sub_db_id(), moveFromLid), diff --git a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp index e9b855e724f..a9fab85ae0d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.cpp @@ -24,7 +24,7 @@ MinimalDocumentRetriever::MinimalDocumentRetriever( MinimalDocumentRetriever::~MinimalDocumentRetriever() = default; Document::UP -MinimalDocumentRetriever::getDocument(DocumentIdT lid) const { +MinimalDocumentRetriever::getFullDocument(DocumentIdT lid) const { return _doc_store.read(lid, *_repo); } diff --git a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h index 193803e6e06..d7265915753 100644 --- a/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/server/minimal_document_retriever.h @@ -27,7 +27,7 @@ public: bool hasFields); ~MinimalDocumentRetriever() override; - document::Document::UP getDocument(search::DocumentIdT lid) const override; + document::Document::UP getFullDocument(search::DocumentIdT lid) const override; void visitDocuments(const LidVector & lids, search::IDocumentVisitor & visitor, ReadConsistency) const override; }; } // namespace proton diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index d31157cb11d..53e455ea204 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -8,11 +8,11 @@ #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> +#include <vespa/vespalib/stllike/hash_map.hpp> -#include <vespa/log/bufferedlogger.h> +#include <vespa/log/log.h> LOG_SETUP(".persistence.thread"); namespace storage { |