From 6c04171843d0d9ebc99f6b91303a76c62eb2aef4 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Thu, 7 Jun 2018 07:34:57 +0200 Subject: 1 - Use a backing buffer for the DocumentUpdate that always is source of truth. 2 - Use this buffer for re-serialization. 3 - Make deserialization lazy where possible. Currently lazy on replay and when arriving over the storageapi. Still needs to eager over documentapi. 4 - Deserialize eagerly in the persistence thread since those are plentyfull and not bottlenecked, instead of in the single master thread. Use real repo. --- storage/src/tests/bucketdb/bucketmanagertest.cpp | 1 + .../distributor/externaloperationhandlertest.cpp | 1 + .../distributor/twophaseupdateoperationtest.cpp | 4 ++-- .../src/tests/distributor/updateoperationtest.cpp | 3 +-- .../filestorage/filestormanagertest.cpp | 1 + .../src/tests/persistence/persistencetestutils.cpp | 26 +++++++--------------- storage/src/tests/persistence/testandsettest.cpp | 2 +- .../changedbucketownershiphandlertest.cpp | 18 +++++---------- .../storageserver/documentapiconvertertest.cpp | 8 +++---- 9 files changed, 25 insertions(+), 39 deletions(-) (limited to 'storage/src') diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index d12e6b90b2a..829a080ad01 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -723,6 +723,7 @@ public: auto createUpdateCommand(const document::BucketId& bucket) const { auto update = std::make_shared( + _self._node->getTestDocMan().getTypeRepo(), *_self._node->getTestDocMan().getTypeRepo() .getDocumentType("testdoctype1"), document::DocumentId("id:foo:testdoctype1::bar2")); diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp index 81b0293b0c0..54aca78d13d 100644 --- a/storage/src/tests/distributor/externaloperationhandlertest.cpp +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -200,6 +200,7 @@ std::shared_ptr ExternalOperationHandlerTest::makeUpdateComm const vespalib::string& doc_type, const vespalib::string& id) const { auto update = std::make_shared( + _testDocMan.getTypeRepo(), *_testDocMan.getTypeRepo().getDocumentType(doc_type), document::DocumentId(id)); return std::make_shared( diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index 28602124045..ea2cc00c642 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -294,7 +294,7 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, document::DocumentUpdate::SP update; if (!options._withError) { update = std::make_shared( - *_doc_type, + *_repo, *_doc_type, document::DocumentId(document::DocIdString("test", "test"))); document::FieldUpdate fup(_doc_type->getField("headerval")); fup.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)); @@ -304,7 +304,7 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, // part of the Get. Just a sneaky way to force an eval error. auto* badDocType = _repo->getDocumentType("testdoctype2"); update = std::make_shared( - *badDocType, + *_repo, *badDocType, document::DocumentId(document::DocIdString("test", "test"))); document::FieldUpdate fup(badDocType->getField("onlyinchild")); fup.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)); diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp index 92474728957..9ce862f5db8 100644 --- a/storage/src/tests/distributor/updateoperationtest.cpp +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -75,8 +75,7 @@ std::shared_ptr UpdateOperation_Test::sendUpdate(const std::string& bucketState) { document::DocumentUpdate::SP update( - new document::DocumentUpdate( - *_html_type, + new document::DocumentUpdate(*_repo, *_html_type, document::DocumentId(document::DocIdString("test", "test")))); _bId = getExternalOperationHandler().getBucketId(update->getId()); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 838df87662f..369e820f987 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -2725,6 +2725,7 @@ void FileStorManagerTest::update_command_size_is_added_to_metric() { document::BucketId bucket(16, 4000); createBucket(bucket, 0); auto update = std::make_shared( + _node->getTestDocMan().getTypeRepo(), _node->getTestDocMan().createRandomDocument()->getType(), document::DocumentId("id:foo:testdoctype1::bar")); auto cmd = std::make_shared( diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index f15921e447d..368eae104e8 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -223,16 +223,11 @@ PersistenceTestUtils::doGetOnDisk( } document::DocumentUpdate::SP -PersistenceTestUtils::createBodyUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) +PersistenceTestUtils::createBodyUpdate(const document::DocumentId& docId, const document::FieldValue& updateValue) { - const DocumentType* docType(_env->_component.getTypeRepo() - ->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr assignUpdate( - new document::AssignValueUpdate(updateValue)); + const DocumentType* docType(_env->_component.getTypeRepo()->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_env->_component.getTypeRepo(), *docType, docId)); + std::shared_ptr assignUpdate(new document::AssignValueUpdate(updateValue)); document::FieldUpdate fieldUpdate(docType->getField("content")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); @@ -240,16 +235,11 @@ PersistenceTestUtils::createBodyUpdate( } document::DocumentUpdate::SP -PersistenceTestUtils::createHeaderUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) +PersistenceTestUtils::createHeaderUpdate(const document::DocumentId& docId, const document::FieldValue& updateValue) { - const DocumentType* docType(_env->_component.getTypeRepo() - ->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr assignUpdate( - new document::AssignValueUpdate(updateValue)); + const DocumentType* docType(_env->_component.getTypeRepo()->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_env->_component.getTypeRepo(), *docType, docId)); + std::shared_ptr assignUpdate(new document::AssignValueUpdate(updateValue)); document::FieldUpdate fieldUpdate(docType->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index c729df1e7eb..686e10ba5ef 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -189,7 +189,7 @@ std::unique_ptr TestAndSetTest::conditional_update_test( { putTestDocument(matchingHeader, timestampOne); - auto docUpdate = std::make_shared(testDoc->getType(), testDocId); + auto docUpdate = std::make_shared(_env->_testDocMan.getTypeRepo(), testDoc->getType(), testDocId); auto fieldUpdate = document::FieldUpdate(testDoc->getField("content")); fieldUpdate.addUpdate(document::AssignValueUpdate(NEW_CONTENT)); docUpdate->addUpdate(fieldUpdate); diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 7df598bed97..98ad8761736 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -597,25 +597,19 @@ ChangedBucketOwnershipHandlerTest::testAbortOutdatedPutOperation() void ChangedBucketOwnershipHandlerTest::testAbortOutdatedUpdateCommand() { - const document::DocumentType* docType(_testDocRepo.getTypeRepo() - .getDocumentType("testdoctype1")); + const document::DocumentType* docType(_testDocRepo.getTypeRepo().getDocumentType("testdoctype1")); document::DocumentId docId("id:foo:testdoctype1::bar"); - document::DocumentUpdate::SP update( - std::make_shared(*docType, docId)); - CPPUNIT_ASSERT(changeAbortsMessage( - getBucketToAbort(), update, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage( - getBucketToAllow(), update, api::Timestamp(1234))); + auto update(std::make_shared(_testDocRepo.getTypeRepo(), *docType, docId)); + CPPUNIT_ASSERT(changeAbortsMessage(getBucketToAbort(), update, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage(getBucketToAllow(), update, api::Timestamp(1234))); } void ChangedBucketOwnershipHandlerTest::testAbortOutdatedRemoveCommand() { document::DocumentId docId("id:foo:testdoctype1::bar"); - CPPUNIT_ASSERT(changeAbortsMessage( - getBucketToAbort(), docId, api::Timestamp(1234))); - CPPUNIT_ASSERT(!changeAbortsMessage( - getBucketToAllow(), docId, api::Timestamp(1234))); + CPPUNIT_ASSERT(changeAbortsMessage(getBucketToAbort(), docId, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage(getBucketToAllow(), docId, api::Timestamp(1234))); } void diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 695ae17c5d4..40d561bd589 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -181,7 +181,7 @@ void DocumentApiConverterTest::testForwardedPut() void DocumentApiConverterTest::testUpdate() { - auto update = std::make_shared(_html_type, defaultDocId); + auto update = std::make_shared(*_repo, _html_type, defaultDocId); documentapi::UpdateDocumentMessage updateMsg(update); updateMsg.setOldTimestamp(1234); updateMsg.setNewTimestamp(5678); @@ -327,19 +327,19 @@ DocumentApiConverterTest::testBatchDocumentUpdate() { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1")); - auto update = std::make_shared(_html_type, docId); + auto update = std::make_shared(*_repo, _html_type, docId); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2")); - auto update = std::make_shared(_html_type, docId); + auto update = std::make_shared(*_repo, _html_type, docId); updates.push_back(update); } { document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3")); - auto update = std::make_shared(_html_type, docId); + auto update = std::make_shared(*_repo, _html_type, docId); updates.push_back(update); } -- cgit v1.2.3