diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-01-05 21:37:36 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-01-06 13:51:43 +0000 |
commit | 6a27f0340bdc34b2ce3bbd1db29ca431ba8ed89e (patch) | |
tree | 5c7b2536964bc93e202c834838db8091b62bfabd | |
parent | 4979cb8498a2cd6bb3ba7c48745a1ba7f69e2f5c (diff) |
Simplify DocEntry to get a clean interface with multiple implementations, instead of an mutant.
Also add tests for the different variations a DocEntry can have.
39 files changed, 223 insertions, 133 deletions
diff --git a/persistence/src/tests/spi/clusterstatetest.cpp b/persistence/src/tests/spi/clusterstatetest.cpp index ac67903244f..9d03cb982c8 100644 --- a/persistence/src/tests/spi/clusterstatetest.cpp +++ b/persistence/src/tests/spi/clusterstatetest.cpp @@ -5,6 +5,7 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> +#include <vespa/document/base/testdocman.h> #include <gtest/gtest.h> using storage::spi::test::makeSpiBucket; @@ -260,4 +261,63 @@ TEST(ClusterStateTest, node_maintenance_state_is_set_independent_of_bucket_space EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:m", d, 0, false)); } +TEST(DocEntryTest, test_basics) { + EXPECT_EQ(40u, sizeof(DocEntry)); +} + +TEST(DocEntryTest, test_meta_only) { + DocEntry e(Timestamp(9), 0); + EXPECT_EQ(9, e.getTimestamp()); + EXPECT_FALSE(e.isRemove()); + EXPECT_EQ(40, e.getSize()); + EXPECT_EQ(0, e.getDocumentSize()); + EXPECT_EQ(0, e.getPersistedDocumentSize()); + EXPECT_EQ(nullptr, e.getDocument()); + EXPECT_EQ(nullptr, e.getDocumentId()); + + e.setPersistedDocumentSize(7); + EXPECT_EQ(40, e.getSize()); + EXPECT_EQ(0, e.getDocumentSize()); + EXPECT_EQ(7, e.getPersistedDocumentSize()); + + DocEntry r(Timestamp(666), 1); + EXPECT_EQ(666, r.getTimestamp()); + EXPECT_TRUE(r.isRemove()); +} + +TEST(DocEntryTest, test_docid_only) { + DocEntry e(Timestamp(9), 0, DocumentId("id:test:test::1")); + EXPECT_EQ(9, e.getTimestamp()); + EXPECT_FALSE(e.isRemove()); + EXPECT_EQ(56, e.getSize()); + EXPECT_EQ(16, e.getDocumentSize()); + EXPECT_EQ(16, e.getPersistedDocumentSize()); + EXPECT_EQ(nullptr, e.getDocument()); + EXPECT_NE(nullptr, e.getDocumentId()); + + e.setPersistedDocumentSize(7); + EXPECT_EQ(56, e.getSize()); + EXPECT_EQ(16, e.getDocumentSize()); + EXPECT_EQ(7, e.getPersistedDocumentSize()); + +} + +TEST(DocEntryTest, test_document_only) { + document::TestDocMan testDocMan; + DocEntry e(Timestamp(9), 0, testDocMan.createRandomDocument(0, 1000)); + EXPECT_EQ(9, e.getTimestamp()); + EXPECT_FALSE(e.isRemove()); + EXPECT_EQ(672, e.getSize()); + EXPECT_EQ(632, e.getDocumentSize()); + EXPECT_EQ(632, e.getPersistedDocumentSize()); + EXPECT_NE(nullptr, e.getDocument()); + EXPECT_NE(nullptr, e.getDocumentId()); + + e.setPersistedDocumentSize(7); + EXPECT_EQ(672, e.getSize()); + EXPECT_EQ(632, e.getDocumentSize()); + EXPECT_EQ(7, e.getPersistedDocumentSize()); + +} + } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 810f2ad2356..57b6b282e60 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -5,6 +5,7 @@ #include <vespa/persistence/spi/test.h> #include <vespa/persistence/spi/catchresult.h> #include <vespa/persistence/spi/resource_usage_listener.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> @@ -25,6 +26,7 @@ using document::BucketId; using document::BucketSpace; using document::test::makeBucketSpace; using storage::spi::test::makeSpiBucket; +using storage::spi::test::cloneDocEntry; namespace storage::spi { @@ -183,7 +185,7 @@ getEntriesFromChunks(const std::vector<Chunk>& chunks) std::vector<spi::DocEntry::UP> ret; for (size_t chunk = 0; chunk < chunks.size(); ++chunk) { for (size_t i = 0; i < chunks[chunk]._entries.size(); ++i) { - ret.push_back(DocEntry::UP(chunks[chunk]._entries[i]->clone())); + ret.push_back(cloneDocEntry(*chunks[chunk]._entries[i])); } } std::sort(ret.begin(), @@ -238,8 +240,7 @@ verifyDocs(const std::vector<DocAndTimestamp>& wanted, const std::vector<Chunk>& chunks, const std::set<string>& removes = std::set<string>()) { - std::vector<DocEntry::UP> retrieved( - getEntriesFromChunks(chunks)); + std::vector<DocEntry::UP> retrieved = getEntriesFromChunks(chunks); size_t removeCount = getRemoveEntryCount(retrieved); // Ensure that we've got the correct number of puts and removes EXPECT_EQ(removes.size(), removeCount); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 9d9f31b63a3..142b7faad9d 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -9,6 +9,7 @@ #include <vespa/persistence/spi/i_resource_usage_listener.h> #include <vespa/persistence/spi/resource_usage.h> #include <vespa/persistence/spi/bucketexecutor.h> +#include <vespa/persistence/spi/test.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -24,6 +25,8 @@ using vespalib::make_string; using std::binary_search; using std::lower_bound; using document::FixedBucketSpaces; +using storage::spi::test::cloneDocEntry; +using storage::spi::test::equal; namespace storage::spi::dummy { @@ -162,7 +165,7 @@ BucketContent::insert(DocEntry::SP e) { auto it = lower_bound(_entries.begin(), _entries.end(), e->getTimestamp(), TimestampLess()); if (it != _entries.end()) { if (it->entry->getTimestamp() == e->getTimestamp()) { - if (*it->entry.get() == *e) { + if (equal(*it->entry, *e)) { LOG(debug, "Ignoring duplicate put entry %s", e->toString().c_str()); return; } else { @@ -664,7 +667,7 @@ DummyPersistence::iterate(IteratorId id, uint64_t maxByteSize, Context& ctx) con entries.push_back(std::move(ret)); } else { // Use entry as-is. - entries.push_back(DocEntry::UP(entry->clone())); + entries.push_back(cloneDocEntry(*entry)); ++fastPath; } } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index a7a784d0479..3b6fc9f1449 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -9,6 +9,7 @@ #pragma once #include <vespa/persistence/spi/abstractpersistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/stllike/hash_map.h> diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp index 8669bbf666b..a48f0656b23 100644 --- a/persistence/src/vespa/persistence/spi/docentry.cpp +++ b/persistence/src/vespa/persistence/spi/docentry.cpp @@ -31,7 +31,7 @@ DocEntry::DocEntry(Timestamp t, int metaFlags, const DocumentId& docId) _metaFlags(metaFlags), _persistedDocumentSize(docId.getSerializedSize()), _size(_persistedDocumentSize + sizeof(DocEntry)), - _documentId(new DocumentId(docId)), + _documentId(std::make_unique<DocumentId>(docId)), _document() { } @@ -46,23 +46,6 @@ DocEntry::DocEntry(Timestamp t, int metaFlags) DocEntry::~DocEntry() = default; -DocEntry* -DocEntry::clone() const { - DocEntry* ret; - if (_documentId) { - ret = new DocEntry(_timestamp, _metaFlags, *_documentId); - ret->setPersistedDocumentSize(_persistedDocumentSize); - } else if (_document) { - ret = new DocEntry(_timestamp, _metaFlags, - std::make_unique<Document>(*_document), - _persistedDocumentSize); - } else { - ret = new DocEntry(_timestamp, _metaFlags); - ret->setPersistedDocumentSize(_persistedDocumentSize); - } - return ret; -} - const DocumentId* DocEntry::getDocumentId() const { return (_document ? &_document->getId() : _documentId.get()); @@ -101,48 +84,4 @@ operator << (std::ostream & os, const DocEntry & r) { return os << r.toString(); } -bool -DocEntry::operator==(const DocEntry& entry) const { - if (_timestamp != entry._timestamp) { - return false; - } - - if (_metaFlags != entry._metaFlags) { - return false; - } - - if (_documentId) { - if (!entry._documentId) { - return false; - } - - if (*_documentId != *entry._documentId) { - return false; - } - } else { - if (entry._documentId) { - return false; - } - } - - if (_document) { - if (!entry._document) { - return false; - } - - if (*_document != *entry._document) { - return false; - } - } else { - if (entry._document) { - return false; - } - } - if (_persistedDocumentSize != entry._persistedDocumentSize) { - return false; - } - - return true; -} - } diff --git a/persistence/src/vespa/persistence/spi/docentry.h b/persistence/src/vespa/persistence/spi/docentry.h index 3374ef6c02d..8109c7f04a5 100644 --- a/persistence/src/vespa/persistence/spi/docentry.h +++ b/persistence/src/vespa/persistence/spi/docentry.h @@ -26,12 +26,12 @@ class DocEntry { public: typedef uint32_t SizeType; private: - Timestamp _timestamp; - int _metaFlags; - SizeType _persistedDocumentSize; - SizeType _size; + Timestamp _timestamp; + int _metaFlags; + SizeType _persistedDocumentSize; + SizeType _size; DocumentIdUP _documentId; - DocumentUP _document; + DocumentUP _document; public: using UP = std::unique_ptr<DocEntry>; using SP = std::shared_ptr<DocEntry>; @@ -47,15 +47,17 @@ public: DocEntry(Timestamp t, int metaFlags, const DocumentId& docId); DocEntry(Timestamp t, int metaFlags); + DocEntry(const DocEntry &) = delete; + DocEntry & operator=(const DocEntry &) = delete; + DocEntry(DocEntry &&) = delete; + DocEntry & operator=(DocEntry &&) = delete; ~DocEntry(); - DocEntry* clone() const; const Document* getDocument() const { return _document.get(); } const DocumentId* getDocumentId() const; DocumentUP releaseDocument(); bool isRemove() const { return (_metaFlags & REMOVE_ENTRY); } Timestamp getTimestamp() const { return _timestamp; } int getFlags() const { return _metaFlags; } - void setFlags(int flags) { _metaFlags = flags; } /** * @return In-memory size of this doc entry, including document instance. * In essence: serialized size of document + sizeof(DocEntry). @@ -87,7 +89,6 @@ public: } vespalib::string toString() const; - bool operator==(const DocEntry& entry) const; }; std::ostream & operator << (std::ostream & os, const DocEntry & r); diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index e458d58fe69..a728f93e60a 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "result.h" +#include "docentry.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> #include <ostream> @@ -48,6 +49,24 @@ GetResult::~GetResult() = default; BucketIdListResult::~BucketIdListResult() = default; IterateResult::~IterateResult() = default; +IterateResult::IterateResult(IterateResult &&) noexcept = default; +IterateResult & IterateResult::operator=(IterateResult &&) noexcept = default; + +IterateResult::IterateResult(ErrorType error, const vespalib::string& errorMessage) + : Result(error, errorMessage), + _completed(false) +{ } + + +IterateResult::IterateResult(List entries, bool completed) + : _completed(completed), + _entries(std::move(entries)) +{ } + +IterateResult::List +IterateResult::steal_entries() { + return std::move(_entries); +} } diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index c734a885b12..10c589307ba 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -3,11 +3,12 @@ #include "bucketinfo.h" #include "bucket.h" -#include "docentry.h" #include <vespa/document/bucket/bucketidlist.h> namespace storage::spi { +class DocEntry; + class Result { public: typedef std::unique_ptr<Result> UP; @@ -279,15 +280,12 @@ private: class IterateResult : public Result { public: - typedef std::vector<DocEntry::UP> List; + using List = std::vector<std::unique_ptr<DocEntry>>; /** * Constructor used when there was an error creating the iterator. */ - IterateResult(ErrorType error, const vespalib::string& errorMessage) - : Result(error, errorMessage), - _completed(false) - { } + IterateResult(ErrorType error, const vespalib::string& errorMessage); /** * Constructor used when the iteration was successful. @@ -296,24 +294,21 @@ public: * * @param completed Set to true if iteration has been completed. */ - IterateResult(List entries, bool completed) - : _completed(completed), - _entries(std::move(entries)) - { } + IterateResult(List entries, bool completed); IterateResult(const IterateResult &) = delete; - IterateResult(IterateResult &&rhs) noexcept = default; - IterateResult &operator=(IterateResult &&rhs) noexcept = default; + IterateResult(IterateResult &&rhs) noexcept; + IterateResult &operator=(IterateResult &&rhs) noexcept; ~IterateResult(); const List& getEntries() const { return _entries; } - List steal_entries() { return std::move(_entries); } + List steal_entries(); bool isCompleted() const { return _completed; } private: bool _completed; - std::vector<DocEntry::UP> _entries; + List _entries; }; } diff --git a/persistence/src/vespa/persistence/spi/test.cpp b/persistence/src/vespa/persistence/spi/test.cpp index 32381110630..c61f21a3893 100644 --- a/persistence/src/vespa/persistence/spi/test.cpp +++ b/persistence/src/vespa/persistence/spi/test.cpp @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "test.h" +#include "docentry.h" #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/fieldvalue/document.h> using document::BucketId; using document::BucketSpace; @@ -9,9 +11,49 @@ using document::test::makeBucketSpace; namespace storage::spi::test { -Bucket makeSpiBucket(BucketId bucketId) +Bucket +makeSpiBucket(BucketId bucketId) { return Bucket(document::Bucket(makeBucketSpace(), bucketId)); } +std::unique_ptr<DocEntry> +cloneDocEntry(const DocEntry & e) { + std::unique_ptr<DocEntry> ret; + if (e.getDocument()) { + ret = std::make_unique<DocEntry>(e.getTimestamp(), e.getFlags(), + std::make_unique<Document>(*e.getDocument()), + e.getPersistedDocumentSize()); + } else if (e.getDocumentId()) { + ret = std::make_unique<DocEntry>(e.getTimestamp(), e.getFlags(), *e.getDocumentId()); + ret->setPersistedDocumentSize(e.getPersistedDocumentSize()); + } else { + ret = std::make_unique<DocEntry>(e.getTimestamp(), e.getFlags()); + ret->setPersistedDocumentSize(e.getPersistedDocumentSize()); + } + return ret; +} + +bool +equal(const DocEntry & a, const DocEntry & b) { + if (a.getTimestamp() != b.getTimestamp()) return false; + if (a.getFlags() != b.getFlags()) return false; + if (a.getPersistedDocumentSize() != b.getPersistedDocumentSize()) return false; + + if (a.getDocument()) { + if (!b.getDocument()) return false; + if (*a.getDocument() != *b.getDocument()) return false; + } else { + if (b.getDocument()) return false; + } + if (a.getDocumentId()) { + if (!b.getDocumentId()) return false; + if (*a.getDocumentId() != *b.getDocumentId()) return false; + } else { + if (b.getDocumentId()) return false; + } + + return true; +} + } diff --git a/persistence/src/vespa/persistence/spi/test.h b/persistence/src/vespa/persistence/spi/test.h index af7109ec80c..1660e5f14fd 100644 --- a/persistence/src/vespa/persistence/spi/test.h +++ b/persistence/src/vespa/persistence/spi/test.h @@ -3,11 +3,16 @@ #pragma once #include "bucket.h" +#include <memory> + +namespace storage::spi { class DocEntry; } namespace storage::spi::test { // Helper functions used by unit tests Bucket makeSpiBucket(document::BucketId bucketId); +std::unique_ptr<DocEntry> cloneDocEntry(const DocEntry & entry); +bool equal(const DocEntry & a, const DocEntry & b); } diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index fab46e61494..b93553c3aa3 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -49,6 +49,7 @@ using storage::spi::IterateResult; using storage::spi::Selection; using storage::spi::Timestamp; using storage::spi::test::makeSpiBucket; +using storage::spi::test::equal; using namespace proton; @@ -390,7 +391,7 @@ void checkEntry(const IterateResult &res, size_t idx, const Timestamp ×tamp { ASSERT_LESS(idx, res.getEntries().size()); DocEntry expect(timestamp, flags); - EXPECT_EQUAL(expect, *res.getEntries()[idx]); + EXPECT_TRUE(equal(expect, *res.getEntries()[idx])); EXPECT_EQUAL(getSize(), res.getEntries()[idx]->getSize()); } @@ -398,7 +399,7 @@ void checkEntry(const IterateResult &res, size_t idx, const DocumentId &id, cons { ASSERT_LESS(idx, res.getEntries().size()); DocEntry expect(timestamp, storage::spi::REMOVE_ENTRY, id); - EXPECT_EQUAL(expect, *res.getEntries()[idx]); + EXPECT_TRUE(equal(expect, *res.getEntries()[idx])); EXPECT_EQUAL(getSize(id), res.getEntries()[idx]->getSize()); EXPECT_GREATER(getSize(id), 0u); } @@ -407,7 +408,7 @@ void checkEntry(const IterateResult &res, size_t idx, const Document &doc, const { ASSERT_LESS(idx, res.getEntries().size()); DocEntry expect(timestamp, storage::spi::NONE, Document::UP(doc.clone())); - EXPECT_EQUAL(expect, *res.getEntries()[idx]); + EXPECT_TRUE(equal(expect, *res.getEntries()[idx])); EXPECT_EQUAL(getSize(doc), res.getEntries()[idx]->getSize()); EXPECT_GREATER(getSize(doc), 0u); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index b39b5dc5734..f7b38fa6560 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_iterator.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/searchcore/proton/common/cachedselect.h> #include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/document/select/gid_filter.h> @@ -8,7 +9,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <algorithm> #include <vespa/log/log.h> LOG_SETUP(".proton.persistenceengine.document_iterator"); @@ -23,18 +23,20 @@ namespace proton { namespace { -DocEntry *createDocEntry(Timestamp timestamp, bool removed) { +std::unique_ptr<DocEntry> +createDocEntry(Timestamp timestamp, bool removed) { int flags = removed ? storage::spi::REMOVE_ENTRY : storage::spi::NONE; - return new DocEntry(timestamp, flags); + return std::make_unique<DocEntry>(timestamp, flags); } -DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ssize_t defaultSerializedSize) { +std::unique_ptr<DocEntry> +createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ssize_t defaultSerializedSize) { if (doc) { if (removed) { - return new DocEntry(timestamp, storage::spi::REMOVE_ENTRY, doc->getId()); + return std::make_unique<DocEntry>(timestamp, storage::spi::REMOVE_ENTRY, doc->getId()); } else { ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->serialize().size(); - return new DocEntry(timestamp, storage::spi::NONE, std::move(doc), serializedSize); + return std::make_unique<DocEntry>(timestamp, storage::spi::NONE, std::move(doc), serializedSize); } } else { return createDocEntry(timestamp, removed); @@ -212,7 +214,7 @@ public: if (doc && _fields) { document::FieldSet::stripFields(*doc, *_fields); } - _list.emplace_back(createDocEntry(meta.timestamp, meta.removed, std::move(doc), _defaultSerializedSize)); + _list.push_back(createDocEntry(meta.timestamp, meta.removed, std::move(doc), _defaultSerializedSize)); } } @@ -262,11 +264,12 @@ DocumentIterator::fetchCompleteSource(const IDocumentRetriever & source, Iterate } LOG(debug, "metadata count after filtering: %zu", lidsToFetch.size()); + list.reserve(lidsToFetch.size()); if ( _metaOnly ) { for (uint32_t lid : lidsToFetch) { const search::DocumentMetaData & meta = metaData[lidIndexMap[lid]]; assert(lid == meta.lid); - list.emplace_back(createDocEntry(meta.timestamp, meta.removed)); + list.push_back(createDocEntry(meta.timestamp, meta.removed)); } } else { MatchVisitor visitor(matcher, metaData, lidIndexMap, _fields.get(), list, _defaultSerializedSize); diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 146dcab2ba7..56e20309c17 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -10,6 +10,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/persistence/spi/test.h> #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <functional> using std::unique_ptr; diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 494af8a0eff..1b5f077aaf0 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -16,6 +16,7 @@ #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/gtest/gtest.h> #include <thread> diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index b5161673af3..3d24ee87879 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -6,6 +6,7 @@ #include "bucketownershipnotifier.h" #include "bucketprocessor.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/catchresult.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/update/documentupdate.h> diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.h b/storage/src/vespa/storage/persistence/bucketprocessor.h index c605a9338e3..002cba6c15c 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.h +++ b/storage/src/vespa/storage/persistence/bucketprocessor.h @@ -7,11 +7,14 @@ #pragma once #include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/context.h> +#include <persistence/spi/types.h> namespace document { class FieldSet; } -namespace storage::spi { struct PersistenceProvider; } +namespace storage::spi { + struct PersistenceProvider; + class DocEntry; +} namespace storage { diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 0c9cecdb6a1..aff6332439b 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -6,7 +6,7 @@ #include "apply_bucket_diff_state.h" #include <vespa/storage/persistence/filestorage/mergestatus.h> #include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/persistence/spi/catchresult.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/objects/nbostream.h> diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 5e66b364242..0f427c11eec 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -16,7 +16,6 @@ #include "types.h" #include "merge_bucket_info_syncer.h" #include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/common/cluster_context.h> #include <vespa/storage/common/messagesender.h> @@ -30,9 +29,9 @@ namespace storage { namespace spi { struct PersistenceProvider; class Context; + class DocEntry; } class PersistenceUtil; -class ApplyBucketDiffEntryResult; class ApplyBucketDiffState; class MergeStatus; @@ -117,7 +116,7 @@ private: */ void populateMetaData(const spi::Bucket&, Timestamp maxTimestamp, - std::vector<spi::DocEntry::UP>& entries, + std::vector<std::unique_ptr<spi::DocEntry>> & entries, spi::Context& context) const; Document::UP deserializeDiffDocument( diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index c6f7442cfdc..cf05ccd65b8 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages.h" +#include <vespa/persistence/spi/docentry.h> #include <ostream> -#include <cassert> using document::BucketSpace; diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h index 594506f53ba..86cf1ac3b06 100644 --- a/storage/src/vespa/storage/persistence/messages.h +++ b/storage/src/vespa/storage/persistence/messages.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/storageapi/message/internal.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/selection.h> #include <vespa/persistence/spi/read_consistency.h> @@ -10,6 +9,8 @@ namespace storage { +namespace spi { class DocEntry; } + class GetIterCommand : public api::InternalCommand { private: document::Bucket _bucket; @@ -44,9 +45,10 @@ private: class GetIterReply : public api::InternalReply { private: + using List = std::vector<std::unique_ptr<spi::DocEntry>>; document::Bucket _bucket; - std::vector<spi::DocEntry::UP> _entries; - bool _completed; + List _entries; + bool _completed; public: typedef std::unique_ptr<GetIterReply> UP; @@ -58,13 +60,9 @@ public: document::Bucket getBucket() const override { return _bucket; } - const std::vector<spi::DocEntry::UP>& getEntries() const { - return _entries; - } + const List & getEntries() const { return _entries; } - std::vector<spi::DocEntry::UP>& getEntries() { - return _entries; - } + List & getEntries() { return _entries; } void setCompleted(bool completed = true) { _completed = completed; } bool isCompleted() const { return _completed; } diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 6a2b5e79450..3da6372cda8 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -5,6 +5,7 @@ #include "persistenceutil.h" #include <vespa/document/fieldset/fieldsets.h> #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 752c1175f22..b7344098698 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -2,6 +2,7 @@ #include "provider_error_wrapper.h" #include "persistenceutil.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/util/idestructorcallback.h> namespace storage { diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp index 9a7a451b906..74813e2e891 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp @@ -3,6 +3,7 @@ #include "simplemessagehandler.h" #include "persistenceutil.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsetrepo.h> diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp index bd472868e06..2a9ac635cff 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp +++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp @@ -3,6 +3,7 @@ #include "splitbitdetector.h" #include "bucketprocessor.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/documentid.h> #include <vespa/document/fieldset/fieldsets.h> diff --git a/storage/src/vespa/storage/visiting/countvisitor.cpp b/storage/src/vespa/storage/visiting/countvisitor.cpp index e20699aa799..3971544a9a0 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.cpp +++ b/storage/src/vespa/storage/visiting/countvisitor.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "countvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/util/stringfmt.h> @@ -22,7 +23,7 @@ CountVisitor::CountVisitor(StorageComponent& component, void CountVisitor::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList& entries, HitCounter& hitCounter) { for (size_t i = 0; i < entries.size(); ++i) { diff --git a/storage/src/vespa/storage/visiting/countvisitor.h b/storage/src/vespa/storage/visiting/countvisitor.h index 4c436b3d07c..4120a96225f 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.h +++ b/storage/src/vespa/storage/visiting/countvisitor.h @@ -18,11 +18,11 @@ public: CountVisitor(StorageComponent&, const vdslib::Parameters& params); - virtual void completedVisiting(HitCounter&) override; + void completedVisiting(HitCounter&) override; private: void handleDocuments(const document::BucketId& bucketId, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) override; bool _doScheme; diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp index 704aaa219cb..92c080169ac 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dumpvisitorsingle.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> @@ -16,8 +17,8 @@ DumpVisitorSingle::DumpVisitorSingle(StorageComponent& component, const vdslib:: { } -void DumpVisitorSingle::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, +void DumpVisitorSingle::handleDocuments(const document::BucketId&, + DocEntryList& entries, HitCounter& hitCounter) { LOG(debug, "Visitor %s handling block of %zu documents.", diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h index 81f4ab7e989..c98bad17e84 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h @@ -19,7 +19,7 @@ public: const vdslib::Parameters& params); private: - void handleDocuments(const document::BucketId&, std::vector<spi::DocEntry::UP>&, HitCounter&) override; + void handleDocuments(const document::BucketId&, DocEntryList&, HitCounter&) override; }; struct DumpVisitorSingleFactory : public VisitorFactory { diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index df965f93ae6..1c3c97d7dcb 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -2,6 +2,7 @@ #include "recoveryvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/text/stringtokenizer.h> @@ -31,7 +32,7 @@ RecoveryVisitor::RecoveryVisitor(StorageComponent& component, void RecoveryVisitor::handleDocuments(const document::BucketId& bid, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) { std::lock_guard guard(_mutex); diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index 045c8aeb484..e850eca3f37 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -22,7 +22,7 @@ public: private: void handleDocuments(const document::BucketId& bucketId, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) override; void completedBucket(const document::BucketId&, HitCounter&) override; diff --git a/storage/src/vespa/storage/visiting/reindexing_visitor.cpp b/storage/src/vespa/storage/visiting/reindexing_visitor.cpp index 528f83e29cc..91ed2810420 100644 --- a/storage/src/vespa/storage/visiting/reindexing_visitor.cpp +++ b/storage/src/vespa/storage/visiting/reindexing_visitor.cpp @@ -3,6 +3,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/storage/common/reindexing_constants.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.reindexing_visitor"); @@ -14,7 +15,7 @@ ReindexingVisitor::ReindexingVisitor(StorageComponent& component) { } -void ReindexingVisitor::handleDocuments(const document::BucketId& /*bucketId*/, +void ReindexingVisitor::handleDocuments(const document::BucketId& , std::vector<spi::DocEntry::UP>& entries, HitCounter& hitCounter) { diff --git a/storage/src/vespa/storage/visiting/reindexing_visitor.h b/storage/src/vespa/storage/visiting/reindexing_visitor.h index 9bde9903617..d9e18542818 100644 --- a/storage/src/vespa/storage/visiting/reindexing_visitor.h +++ b/storage/src/vespa/storage/visiting/reindexing_visitor.h @@ -20,7 +20,7 @@ public: ~ReindexingVisitor() override = default; private: - void handleDocuments(const document::BucketId&, std::vector<spi::DocEntry::UP>&, HitCounter&) override; + void handleDocuments(const document::BucketId&, DocEntryList&, HitCounter&) override; bool remap_docapi_message_error_code(api::ReturnCode& in_out_code) override; vespalib::string make_lock_access_token() const; }; diff --git a/storage/src/vespa/storage/visiting/testvisitor.cpp b/storage/src/vespa/storage/visiting/testvisitor.cpp index b5ace673e48..10c79ea6f18 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.cpp +++ b/storage/src/vespa/storage/visiting/testvisitor.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <sstream> @@ -39,7 +40,7 @@ TestVisitor::startingVisitor(const std::vector<document::BucketId>& buckets) void TestVisitor::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& /*hitCounter*/) { std::ostringstream ost; diff --git a/storage/src/vespa/storage/visiting/testvisitor.h b/storage/src/vespa/storage/visiting/testvisitor.h index 482b1347faa..989581ac121 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.h +++ b/storage/src/vespa/storage/visiting/testvisitor.h @@ -20,7 +20,7 @@ private: void startingVisitor(const std::vector<document::BucketId>& buckets) override; void handleDocuments(const document::BucketId& bucketId, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) override; void completedBucket(const document::BucketId& bucket, HitCounter& hitCounter) override; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index dfb78122e07..7da397b670a 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -2,6 +2,7 @@ #include "visitor.h" #include "visitormetrics.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 52b2d586c78..8857a54e8df 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -18,7 +18,6 @@ #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/visitorfactory.h> #include <vespa/documentapi/messagebus/messages/documentmessage.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/selection.h> #include <vespa/persistence/spi/read_consistency.h> #include <list> @@ -39,6 +38,10 @@ namespace documentapi { namespace storage { +namespace spi { + class DocEntry; +} + namespace api { class ReturnCode; class StorageCommand; @@ -357,6 +360,7 @@ protected: // error code, false if the DocumentAPI message should be retried later. [[nodiscard]] virtual bool remap_docapi_message_error_code(api::ReturnCode& in_out_code); public: + using DocEntryList = std::vector<std::unique_ptr<spi::DocEntry>>; Visitor(StorageComponent& component); virtual ~Visitor(); @@ -398,7 +402,7 @@ public: * vector of documents arrive from the persistence layer. */ virtual void handleDocuments(const document::BucketId&, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) = 0; /** diff --git a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp index b6244521a46..104af486c90 100644 --- a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp +++ b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp @@ -10,6 +10,8 @@ #include <vespa/searchvisitor/searchvisitor.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/persistence/spi/docentry.h> + #include <vespa/log/log.h> LOG_SETUP("searchvisitor_test"); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index cc6323c3053..460a0886ee2 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -3,6 +3,7 @@ #include "querytermdata.h" #include "searchenvironment.h" #include "searchvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/datatype/weightedsetdatatype.h> @@ -864,7 +865,7 @@ SearchVisitor::compatibleDocumentTypes(const document::DocumentType& typeA, void SearchVisitor::handleDocuments(const document::BucketId&, - std::vector<storage::spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) { (void) hitCounter; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index dff263b8418..20ab1ccf325 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -298,7 +298,7 @@ private: // Inherit doc from Visitor void handleDocuments(const document::BucketId&, - std::vector<storage::spi::DocEntry::UP>& entries, + DocEntryList& entries, HitCounter& hitCounter) override; bool compatibleDocumentTypes(const document::DocumentType& typeA, |