diff options
author | Henning Baldersheim <balder@oath.com> | 2018-06-07 07:34:57 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@oath.com> | 2018-06-12 22:50:28 +0200 |
commit | 6c04171843d0d9ebc99f6b91303a76c62eb2aef4 (patch) | |
tree | 338ffa8e190c93ef4abab1ebd1ac79036c74cfd2 /searchcore | |
parent | 047ea0bed80966554e29a1db1b35361d946a3866 (diff) |
1 - Use a backing buffer for the DocumentUpdate that always is source of truth.
2 - Use this buffer for re-serialization.
3 - Make deserialization lazy where possible.
Currently lazy on replay and when arriving over the storageapi.
Still needs to eager over documentapi.
4 - Deserialize eagerly in the persistence thread since those are plentyfull and not bottlenecked,
instead of in the single master thread.
Use real repo.
Diffstat (limited to 'searchcore')
8 files changed, 23 insertions, 20 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index c41b6bdcb97..0e6bd2c74fe 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -445,7 +445,7 @@ TEST_F("require that attribute writer handles update", Fixture) schema.addAttributeField(Schema::AttributeField("a2", schema::DataType::INT32, CollectionType::SINGLE)); DocBuilder idb(schema); const document::DocumentType &dt(idb.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("doc::1")); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 5))); upd.addUpdate(FieldUpdate(upd.getType().getField("a2")) @@ -489,7 +489,7 @@ TEST_F("require that attribute writer handles predicate update", Fixture) EXPECT_EQUAL(2u, a1->getNumDocs()); const document::DocumentType &dt(idb.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*idb.getDocumentTypeRepo(), dt, DocumentId("doc::1")); PredicateFieldValue new_value(builder.feature("foo").value("bar").build()); upd.addUpdate(FieldUpdate(upd.getType().getField("a1")) .addUpdate(AssignValueUpdate(new_value))); @@ -678,7 +678,7 @@ TEST_F("require that attribute writer handles tensor assign update", Fixture) EXPECT_TRUE(tensor->equals(*tensor2)); const document::DocumentType &dt(builder.getDocumentType()); - DocumentUpdate upd(dt, DocumentId("doc::1")); + DocumentUpdate upd(*builder.getDocumentTypeRepo(), dt, DocumentId("doc::1")); auto new_tensor = createTensor({ {{{"x", "8"}, {"y", "9"}}, 11} }, {"x", "y"}); TensorFieldValue new_value; diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index e774728b41e..a1bce7174bf 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -316,7 +316,7 @@ struct UpdateContext { DocumentUpdate::SP update; BucketId bucketId; UpdateContext(const vespalib::string &docId, DocBuilder &builder) : - update(new DocumentUpdate(builder.getDocumentType(), DocumentId(docId))), + update(new DocumentUpdate(*builder.getDocumentTypeRepo(), builder.getDocumentType(), DocumentId(docId))), bucketId(BucketFactory::getBucketId(update->getId())) { } diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index a9fa79f513c..111b4047da3 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -478,11 +478,11 @@ struct DocumentContext DocumentContext::DocumentContext(const vespalib::string &docId, uint64_t timestamp, DocBuilder &builder) : doc(builder.startDocument(docId).startSummaryField("s1").addStr(docId).endField().endDocument().release()), - upd(new DocumentUpdate(builder.getDocumentType(), doc->getId())), + upd(new DocumentUpdate(*builder.getDocumentTypeRepo(), builder.getDocumentType(), doc->getId())), bid(BucketFactory::getNumBucketBits(), doc->getId().getGlobalId().convertToBucketId().getRawId()), ts(timestamp) {} -DocumentContext::~DocumentContext() {} +DocumentContext::~DocumentContext() = default; struct FeedTokenContext { diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index 9a9d896f2b9..e87e9209a17 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -122,7 +122,7 @@ public: } auto makeUpdate() { - auto upd(std::make_shared<DocumentUpdate>(_docType, docId)); + auto upd(std::make_shared<DocumentUpdate>(*_repo, _docType, docId)); upd->addUpdate(FieldUpdate(upd->getType().getField("string")). addUpdate(AssignValueUpdate(StringFieldValue("newval")))); return upd; @@ -136,6 +136,7 @@ public: TEST("require that toString() on derived classes are meaningful") { + DocumentTypeRepo repo; BucketId bucket_id1(42); BucketId bucket_id2(43); BucketId bucket_id3(44); @@ -146,7 +147,7 @@ TEST("require that toString() on derived classes are meaningful") MyStreamHandler stream_handler; DocumentIdT doc_id_limit = 15; DocumentId doc_id("doc:foo:bar"); - DocumentUpdate::SP update(new DocumentUpdate(*DataType::DOCUMENT, doc_id)); + DocumentUpdate::SP update(new DocumentUpdate(repo, *DataType::DOCUMENT, doc_id)); EXPECT_EQUAL("DeleteBucket(BucketId(0x0000000000000000), serialNum=0)", DeleteBucketOperation().toString()); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 73580416e65..7f065d0cc15 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -67,7 +67,9 @@ createDoc(const DocumentType &docType, const DocumentId &docId) document::DocumentUpdate::SP createUpd(const DocumentType& docType, const DocumentId &docId) { - return document::DocumentUpdate::SP(new document::DocumentUpdate(docType, docId)); + static std::vector<std::unique_ptr<document::DocumentTypeRepo>> repoList; + repoList.emplace_back(std::make_unique<document::DocumentTypeRepo>(docType)); + return std::make_shared<document::DocumentUpdate>(*repoList.back(), docType, docId); } storage::spi::ClusterState diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index d31f1faec77..824909d0431 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -51,12 +51,11 @@ UpdateOperation::serializeUpdate(vespalib::nbostream &os) const } void -UpdateOperation::deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) +UpdateOperation::deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo) { - document::ByteBuffer buf(is.peek(), is.size()); - DocumentUpdate::UP update = (getType() == UPDATE_42) ? DocumentUpdate::create42(repo, buf) : DocumentUpdate::createHEAD(repo, buf); - is.adjustReadPos(buf.getPos()); - _upd = std::move(update); + _upd = (getType() == UPDATE_42) + ? DocumentUpdate::create42(repo, is) + : DocumentUpdate::createHEAD(repo, std::move(is)); } void @@ -73,7 +72,7 @@ UpdateOperation::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &re { DocumentOperation::deserialize(is, repo); try { - deserializeUpdate(is, repo); + deserializeUpdate(std::move(is), repo); } catch (document::DocumentTypeNotFoundException &e) { LOG(warning, "Failed deserialize update operation using unknown document type '%s'", e.getDocumentTypeName().c_str()); @@ -87,14 +86,14 @@ UpdateOperation::deserializeUpdate(const DocumentTypeRepo &repo) { vespalib::nbostream stream; serializeUpdate(stream); - deserializeUpdate(stream, repo); + deserializeUpdate(std::move(stream), repo); } -vespalib::string UpdateOperation::toString() const { +vespalib::string +UpdateOperation::toString() const { return make_string("%s(%s, %s)", ((getType() == FeedOperation::UPDATE_42) ? "Update42" : "Update"), - _upd.get() ? - _upd->getId().getScheme().toString().c_str() : "NULL", + _upd.get() ? _upd->getId().getScheme().toString().c_str() : "NULL", docArgsToString().c_str()); } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 99dcbfbce6c..554778728f0 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -19,7 +19,7 @@ private: const storage::spi::Timestamp ×tamp, const DocumentUpdateSP &upd); void serializeUpdate(vespalib::nbostream &os) const; - void deserializeUpdate(vespalib::nbostream &is, const document::DocumentTypeRepo &repo); + void deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo); public: UpdateOperation(); UpdateOperation(Type type); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index bab3494ca5c..f774c953af4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -377,6 +377,7 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP upd->getId().toString().c_str(), state.message().c_str())); } } + upd->eagerDeserialize(); std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); DocTypeName docType(upd->getType()); LOG(spam, "update(%s, %" PRIu64 ", (\"%s\", \"%s\"), createIfNonExistent='%s')", |